diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,84984 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.9999146539216524, + "eval_steps": 500, + "global_step": 2929, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.587890625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 1698.599609375, + "completions/mean_terminated_length": 1200.1658935546875, + "completions/min_length": 20.0, + "completions/min_terminated_length": 20.0, + "epoch": 0.0003413843133907997, + "grad_norm": 0.10431239753961563, + "kl": 0.0006151199340820312, + "learning_rate": 0.0, + "loss": 0.0844, + "num_tokens": 948291.0, + "reward": 0.46240234375, + "reward_std": 0.2197888195514679, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.39208984375, + "rewards/tag_count_reward/std": 0.18950168788433075, + "step": 1 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.673828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2025.0, + "completions/mean_length": 1758.5859375, + "completions/mean_terminated_length": 1160.6947021484375, + "completions/min_length": 112.0, + "completions/min_terminated_length": 112.0, + "epoch": 0.0006827686267815994, + "grad_norm": 0.09877148270606995, + "kl": 0.0005578994750976562, + "learning_rate": 3.424657534246575e-09, + "loss": 0.0661, + "num_tokens": 1921919.0, + "reward": 0.3974609375, + "reward_std": 0.1725420355796814, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.3583984375, + "rewards/tag_count_reward/std": 0.15551748871803284, + "step": 2 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.529296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2031.0, + "completions/mean_length": 1627.919921875, + "completions/mean_terminated_length": 1155.5477294921875, + "completions/min_length": 20.0, + "completions/min_terminated_length": 20.0, + "epoch": 0.001024152940172399, + "grad_norm": 0.092210553586483, + "kl": 0.0005474090576171875, + "learning_rate": 6.84931506849315e-09, + "loss": 0.074, + "num_tokens": 2837286.0, + "reward": 0.50048828125, + "reward_std": 0.21651187539100647, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.40869140625, + "rewards/tag_count_reward/std": 0.18128602206707, + "step": 3 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.673828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 1769.169921875, + "completions/mean_terminated_length": 1193.143798828125, + "completions/min_length": 32.0, + "completions/min_terminated_length": 32.0, + "epoch": 0.0013655372535631989, + "grad_norm": 0.09093520045280457, + "kl": 0.000579833984375, + "learning_rate": 1.0273972602739724e-08, + "loss": 0.0932, + "num_tokens": 3823501.0, + "reward": 0.38232421875, + "reward_std": 0.1507340669631958, + "rewards/accuracy_reward/mean": 0.03125, + "rewards/accuracy_reward/std": 0.17416280508041382, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.35107421875, + "rewards/tag_count_reward/std": 0.15216577053070068, + "step": 4 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.51171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 1616.763671875, + "completions/mean_terminated_length": 1164.8280029296875, + "completions/min_length": 231.0, + "completions/min_terminated_length": 231.0, + "epoch": 0.0017069215669539984, + "grad_norm": 0.11474630981683731, + "kl": 0.0005540847778320312, + "learning_rate": 1.36986301369863e-08, + "loss": 0.0551, + "num_tokens": 4732788.0, + "reward": 0.5478515625, + "reward_std": 0.23933620750904083, + "rewards/accuracy_reward/mean": 0.14314515888690948, + "rewards/accuracy_reward/std": 0.35057440400123596, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.4091796875, + "rewards/tag_count_reward/std": 0.17640604078769684, + "step": 5 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 1634.732421875, + "completions/mean_terminated_length": 1119.9605712890625, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "epoch": 0.002048305880344798, + "grad_norm": 0.11417374759912491, + "kl": 0.00060272216796875, + "learning_rate": 1.7123287671232876e-08, + "loss": 0.0843, + "num_tokens": 5657051.0, + "reward": 0.4462890625, + "reward_std": 0.17480555176734924, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.3837890625, + "rewards/tag_count_reward/std": 0.1545807421207428, + "step": 6 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.603515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 1716.001953125, + "completions/mean_terminated_length": 1210.645263671875, + "completions/min_length": 218.0, + "completions/min_terminated_length": 218.0, + "epoch": 0.0023896901937355977, + "grad_norm": 0.10724660009145737, + "kl": 0.0005521774291992188, + "learning_rate": 2.054794520547945e-08, + "loss": 0.0957, + "num_tokens": 6608028.0, + "reward": 0.4287109375, + "reward_std": 0.19955545663833618, + "rewards/accuracy_reward/mean": 0.04233871027827263, + "rewards/accuracy_reward/std": 0.2015640139579773, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.3876953125, + "rewards/tag_count_reward/std": 0.1806028187274933, + "step": 7 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.55078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 1729.3828125, + "completions/mean_terminated_length": 1338.7303466796875, + "completions/min_length": 310.0, + "completions/min_terminated_length": 310.0, + "epoch": 0.0027310745071263977, + "grad_norm": 0.09726861119270325, + "kl": 0.00057220458984375, + "learning_rate": 2.3972602739726024e-08, + "loss": 0.0874, + "num_tokens": 7570176.0, + "reward": 0.5234375, + "reward_std": 0.27367502450942993, + "rewards/accuracy_reward/mean": 0.126953125, + "rewards/accuracy_reward/std": 0.33324605226516724, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.396484375, + "rewards/tag_count_reward/std": 0.16633892059326172, + "step": 8 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.556640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2025.0, + "completions/mean_length": 1642.70703125, + "completions/mean_terminated_length": 1133.8590087890625, + "completions/min_length": 147.0, + "completions/min_terminated_length": 147.0, + "epoch": 0.0030724588205171973, + "grad_norm": 0.10777005553245544, + "kl": 0.0005903244018554688, + "learning_rate": 2.73972602739726e-08, + "loss": 0.0881, + "num_tokens": 8485034.0, + "reward": 0.453125, + "reward_std": 0.20808660984039307, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.404296875, + "rewards/tag_count_reward/std": 0.1943957805633545, + "step": 9 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 1675.646484375, + "completions/mean_terminated_length": 1148.731201171875, + "completions/min_length": 276.0, + "completions/min_terminated_length": 276.0, + "epoch": 0.003413843133907997, + "grad_norm": 0.0893247202038765, + "kl": 0.0005788803100585938, + "learning_rate": 3.082191780821918e-08, + "loss": 0.089, + "num_tokens": 9414901.0, + "reward": 0.41845703125, + "reward_std": 0.1757884919643402, + "rewards/accuracy_reward/mean": 0.03515625, + "rewards/accuracy_reward/std": 0.1843547374010086, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.38330078125, + "rewards/tag_count_reward/std": 0.16157081723213196, + "step": 10 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.576171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 1673.0078125, + "completions/mean_terminated_length": 1163.225830078125, + "completions/min_length": 171.0, + "completions/min_terminated_length": 171.0, + "epoch": 0.003755227447298797, + "grad_norm": 0.10025846213102341, + "kl": 0.0005784034729003906, + "learning_rate": 3.424657534246575e-08, + "loss": 0.0832, + "num_tokens": 10347065.0, + "reward": 0.431640625, + "reward_std": 0.19180549681186676, + "rewards/accuracy_reward/mean": 0.037109375, + "rewards/accuracy_reward/std": 0.18921469151973724, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.39453125, + "rewards/tag_count_reward/std": 0.17862646281719208, + "step": 11 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.517578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2016.0, + "completions/mean_length": 1657.361328125, + "completions/mean_terminated_length": 1238.255126953125, + "completions/min_length": 4.0, + "completions/min_terminated_length": 4.0, + "epoch": 0.004096611760689596, + "grad_norm": 0.09967684745788574, + "kl": 0.0006275177001953125, + "learning_rate": 3.767123287671233e-08, + "loss": 0.1118, + "num_tokens": 11267010.0, + "reward": 0.50390625, + "reward_std": 0.24697786569595337, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.41015625, + "rewards/tag_count_reward/std": 0.17411890625953674, + "step": 12 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.576171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2009.0, + "completions/mean_length": 1611.318359375, + "completions/mean_terminated_length": 1017.6727905273438, + "completions/min_length": 18.0, + "completions/min_terminated_length": 18.0, + "epoch": 0.004437996074080396, + "grad_norm": 0.11828067153692245, + "kl": 0.0007390975952148438, + "learning_rate": 4.10958904109589e-08, + "loss": 0.0974, + "num_tokens": 12167813.0, + "reward": 0.474609375, + "reward_std": 0.20802630484104156, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.39453125, + "rewards/tag_count_reward/std": 0.19436629116535187, + "step": 13 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.51171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 1670.515625, + "completions/mean_terminated_length": 1274.912109375, + "completions/min_length": 98.0, + "completions/min_terminated_length": 98.0, + "epoch": 0.0047793803874711955, + "grad_norm": 0.0998254269361496, + "kl": 0.0006895065307617188, + "learning_rate": 4.452054794520547e-08, + "loss": 0.108, + "num_tokens": 13100381.0, + "reward": 0.46630859375, + "reward_std": 0.22284796833992004, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.40966796875, + "rewards/tag_count_reward/std": 0.17561545968055725, + "step": 14 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.486328125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 1561.16015625, + "completions/mean_terminated_length": 1100.2357177734375, + "completions/min_length": 14.0, + "completions/min_terminated_length": 14.0, + "epoch": 0.005120764700861995, + "grad_norm": 0.10169485211372375, + "kl": 0.0006809234619140625, + "learning_rate": 4.794520547945205e-08, + "loss": 0.1075, + "num_tokens": 13976847.0, + "reward": 0.5224609375, + "reward_std": 0.2715893089771271, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.4306640625, + "rewards/tag_count_reward/std": 0.20048168301582336, + "step": 15 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.57421875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2025.0, + "completions/mean_length": 1646.515625, + "completions/mean_terminated_length": 1105.064208984375, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "epoch": 0.0054621490142527955, + "grad_norm": 0.1059543713927269, + "kl": 0.00087738037109375, + "learning_rate": 5.136986301369862e-08, + "loss": 0.0657, + "num_tokens": 14895031.0, + "reward": 0.50146484375, + "reward_std": 0.2400987446308136, + "rewards/accuracy_reward/mean": 0.1171875, + "rewards/accuracy_reward/std": 0.32195815443992615, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.38427734375, + "rewards/tag_count_reward/std": 0.16227306425571442, + "step": 16 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.486328125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 1559.234375, + "completions/mean_terminated_length": 1096.4866943359375, + "completions/min_length": 23.0, + "completions/min_terminated_length": 23.0, + "epoch": 0.005803533327643595, + "grad_norm": 0.10745397210121155, + "kl": 0.00081634521484375, + "learning_rate": 5.47945205479452e-08, + "loss": 0.0947, + "num_tokens": 15769087.0, + "reward": 0.53662109375, + "reward_std": 0.2537875771522522, + "rewards/accuracy_reward/mean": 0.107421875, + "rewards/accuracy_reward/std": 0.30995169281959534, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.42919921875, + "rewards/tag_count_reward/std": 0.20209747552871704, + "step": 17 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.609375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 1708.259765625, + "completions/mean_terminated_length": 1178.2650146484375, + "completions/min_length": 218.0, + "completions/min_terminated_length": 218.0, + "epoch": 0.006144917641034395, + "grad_norm": 0.10190171003341675, + "kl": 0.0008611679077148438, + "learning_rate": 5.821917808219177e-08, + "loss": 0.115, + "num_tokens": 16726676.0, + "reward": 0.45458984375, + "reward_std": 0.20294985175132751, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.39013671875, + "rewards/tag_count_reward/std": 0.19725459814071655, + "step": 18 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.56640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 1668.79296875, + "completions/mean_terminated_length": 1173.4324951171875, + "completions/min_length": 250.0, + "completions/min_terminated_length": 250.0, + "epoch": 0.006486301954425194, + "grad_norm": 0.09239482879638672, + "kl": 0.0009212493896484375, + "learning_rate": 6.164383561643836e-08, + "loss": 0.0917, + "num_tokens": 17658442.0, + "reward": 0.4638671875, + "reward_std": 0.1933201402425766, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.3974609375, + "rewards/tag_count_reward/std": 0.1863318681716919, + "step": 19 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.529296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 1626.255859375, + "completions/mean_terminated_length": 1152.012451171875, + "completions/min_length": 18.0, + "completions/min_terminated_length": 18.0, + "epoch": 0.006827686267815994, + "grad_norm": 0.11028622090816498, + "kl": 0.0010242462158203125, + "learning_rate": 6.506849315068492e-08, + "loss": 0.1159, + "num_tokens": 18568301.0, + "reward": 0.521484375, + "reward_std": 0.2340538650751114, + "rewards/accuracy_reward/mean": 0.107421875, + "rewards/accuracy_reward/std": 0.30995169281959534, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.4140625, + "rewards/tag_count_reward/std": 0.20196563005447388, + "step": 20 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 1615.5078125, + "completions/mean_terminated_length": 1169.2857666015625, + "completions/min_length": 166.0, + "completions/min_terminated_length": 166.0, + "epoch": 0.007169070581206793, + "grad_norm": 0.10006025433540344, + "kl": 0.0010442733764648438, + "learning_rate": 6.84931506849315e-08, + "loss": 0.0826, + "num_tokens": 19468801.0, + "reward": 0.5556640625, + "reward_std": 0.2516134977340698, + "rewards/accuracy_reward/mean": 0.140625, + "rewards/accuracy_reward/std": 0.3479743003845215, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.4150390625, + "rewards/tag_count_reward/std": 0.19436383247375488, + "step": 21 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.529296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 1643.8984375, + "completions/mean_terminated_length": 1189.493896484375, + "completions/min_length": 190.0, + "completions/min_terminated_length": 190.0, + "epoch": 0.007510454894597594, + "grad_norm": 0.1029755026102066, + "kl": 0.0013523101806640625, + "learning_rate": 7.191780821917807e-08, + "loss": 0.0927, + "num_tokens": 20393389.0, + "reward": 0.537109375, + "reward_std": 0.2739160656929016, + "rewards/accuracy_reward/mean": 0.12890625, + "rewards/accuracy_reward/std": 0.33542385697364807, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.408203125, + "rewards/tag_count_reward/std": 0.19122402369976044, + "step": 22 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 1683.533203125, + "completions/mean_terminated_length": 1184.0787353515625, + "completions/min_length": 335.0, + "completions/min_terminated_length": 335.0, + "epoch": 0.007851839207988393, + "grad_norm": 0.10230695456266403, + "kl": 0.0011806488037109375, + "learning_rate": 7.534246575342466e-08, + "loss": 0.1142, + "num_tokens": 21327022.0, + "reward": 0.486328125, + "reward_std": 0.27032989263534546, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.412109375, + "rewards/tag_count_reward/std": 0.21004575490951538, + "step": 23 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.478515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 1499.361328125, + "completions/mean_terminated_length": 995.9288330078125, + "completions/min_length": 121.0, + "completions/min_terminated_length": 121.0, + "epoch": 0.008193223521379193, + "grad_norm": 0.11834343522787094, + "kl": 0.0013599395751953125, + "learning_rate": 7.876712328767122e-08, + "loss": 0.129, + "num_tokens": 22174311.0, + "reward": 0.49951171875, + "reward_std": 0.22222651541233063, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.44287109375, + "rewards/tag_count_reward/std": 0.2110753357410431, + "step": 24 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.51171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 1673.552734375, + "completions/mean_terminated_length": 1281.132080078125, + "completions/min_length": 299.0, + "completions/min_terminated_length": 299.0, + "epoch": 0.008534607834769992, + "grad_norm": 0.09323837608098984, + "kl": 0.0014896392822265625, + "learning_rate": 8.21917808219178e-08, + "loss": 0.0923, + "num_tokens": 23108274.0, + "reward": 0.470703125, + "reward_std": 0.22797748446464539, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.427734375, + "rewards/tag_count_reward/std": 0.20607776939868927, + "step": 25 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.416015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 1454.22265625, + "completions/mean_terminated_length": 1031.230712890625, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "epoch": 0.008875992148160792, + "grad_norm": 0.11998679488897324, + "kl": 0.0018024444580078125, + "learning_rate": 8.561643835616438e-08, + "loss": 0.0999, + "num_tokens": 23931380.0, + "reward": 0.64453125, + "reward_std": 0.3692547380924225, + "rewards/accuracy_reward/mean": 0.19153225421905518, + "rewards/accuracy_reward/std": 0.3939041793346405, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.458984375, + "rewards/tag_count_reward/std": 0.2099001258611679, + "step": 26 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.443359375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2020.0, + "completions/mean_length": 1422.546875, + "completions/mean_terminated_length": 924.3789672851562, + "completions/min_length": 4.0, + "completions/min_terminated_length": 4.0, + "epoch": 0.009217376461551591, + "grad_norm": 0.12516747415065765, + "kl": 0.0019092559814453125, + "learning_rate": 8.904109589041094e-08, + "loss": 0.1022, + "num_tokens": 24729004.0, + "reward": 0.51806640625, + "reward_std": 0.24743953347206116, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.46337890625, + "rewards/tag_count_reward/std": 0.2377152144908905, + "step": 27 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.517578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 1586.744140625, + "completions/mean_terminated_length": 1091.87451171875, + "completions/min_length": 14.0, + "completions/min_terminated_length": 14.0, + "epoch": 0.009558760774942391, + "grad_norm": 0.11038284003734589, + "kl": 0.002349853515625, + "learning_rate": 9.246575342465753e-08, + "loss": 0.1299, + "num_tokens": 25630505.0, + "reward": 0.50927734375, + "reward_std": 0.25093698501586914, + "rewards/accuracy_reward/mean": 0.058467742055654526, + "rewards/accuracy_reward/std": 0.23486268520355225, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.45068359375, + "rewards/tag_count_reward/std": 0.2440776377916336, + "step": 28 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.486328125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2041.0, + "completions/mean_length": 1575.810546875, + "completions/mean_terminated_length": 1128.756591796875, + "completions/min_length": 154.0, + "completions/min_terminated_length": 154.0, + "epoch": 0.00990014508833319, + "grad_norm": 0.11193142831325531, + "kl": 0.001926422119140625, + "learning_rate": 9.58904109589041e-08, + "loss": 0.1319, + "num_tokens": 26512584.0, + "reward": 0.48291015625, + "reward_std": 0.23605626821517944, + "rewards/accuracy_reward/mean": 0.04435483738780022, + "rewards/accuracy_reward/std": 0.2060900777578354, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.43994140625, + "rewards/tag_count_reward/std": 0.2148621678352356, + "step": 29 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.462890625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 1476.640625, + "completions/mean_terminated_length": 984.2327270507812, + "completions/min_length": 232.0, + "completions/min_terminated_length": 232.0, + "epoch": 0.01024152940172399, + "grad_norm": 0.11076265573501587, + "kl": 0.002689361572265625, + "learning_rate": 9.931506849315068e-08, + "loss": 0.1011, + "num_tokens": 27348592.0, + "reward": 0.60107421875, + "reward_std": 0.27622419595718384, + "rewards/accuracy_reward/mean": 0.1328125, + "rewards/accuracy_reward/std": 0.33970388770103455, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.46826171875, + "rewards/tag_count_reward/std": 0.23790405690670013, + "step": 30 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.443359375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 1510.275390625, + "completions/mean_terminated_length": 1081.982421875, + "completions/min_length": 130.0, + "completions/min_terminated_length": 130.0, + "epoch": 0.01058291371511479, + "grad_norm": 0.11147204041481018, + "kl": 0.002384185791015625, + "learning_rate": 1.0273972602739725e-07, + "loss": 0.1056, + "num_tokens": 28200637.0, + "reward": 0.58056640625, + "reward_std": 0.2617027759552002, + "rewards/accuracy_reward/mean": 0.11328125, + "rewards/accuracy_reward/std": 0.3172462284564972, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.46728515625, + "rewards/tag_count_reward/std": 0.23151643574237823, + "step": 31 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 1437.0859375, + "completions/mean_terminated_length": 1111.509033203125, + "completions/min_length": 155.0, + "completions/min_terminated_length": 155.0, + "epoch": 0.010924298028505591, + "grad_norm": 0.11107554286718369, + "kl": 0.002471923828125, + "learning_rate": 1.0616438356164383e-07, + "loss": 0.1385, + "num_tokens": 29013849.0, + "reward": 0.61376953125, + "reward_std": 0.3043445944786072, + "rewards/accuracy_reward/mean": 0.123046875, + "rewards/accuracy_reward/std": 0.32881227135658264, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.49072265625, + "rewards/tag_count_reward/std": 0.22182057797908783, + "step": 32 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3984375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 1431.798828125, + "completions/mean_terminated_length": 1023.6655883789062, + "completions/min_length": 4.0, + "completions/min_terminated_length": 4.0, + "epoch": 0.01126568234189639, + "grad_norm": 0.12299497425556183, + "kl": 0.002918243408203125, + "learning_rate": 1.095890410958904e-07, + "loss": 0.1306, + "num_tokens": 29831666.0, + "reward": 0.5302734375, + "reward_std": 0.26406511664390564, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.4833984375, + "rewards/tag_count_reward/std": 0.24020493030548096, + "step": 33 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 1359.208984375, + "completions/mean_terminated_length": 1034.6063232421875, + "completions/min_length": 174.0, + "completions/min_terminated_length": 174.0, + "epoch": 0.01160706665528719, + "grad_norm": 0.1284693032503128, + "kl": 0.003650665283203125, + "learning_rate": 1.1301369863013698e-07, + "loss": 0.1532, + "num_tokens": 30600861.0, + "reward": 0.64892578125, + "reward_std": 0.31966668367385864, + "rewards/accuracy_reward/mean": 0.12890625, + "rewards/accuracy_reward/std": 0.33542385697364807, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.52001953125, + "rewards/tag_count_reward/std": 0.23763883113861084, + "step": 34 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.37890625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 1410.498046875, + "completions/mean_terminated_length": 1021.5817260742188, + "completions/min_length": 163.0, + "completions/min_terminated_length": 163.0, + "epoch": 0.01194845096867799, + "grad_norm": 0.11522830277681351, + "kl": 0.0032196044921875, + "learning_rate": 1.1643835616438355e-07, + "loss": 0.1269, + "num_tokens": 31405052.0, + "reward": 0.67529296875, + "reward_std": 0.335245817899704, + "rewards/accuracy_reward/mean": 0.169921875, + "rewards/accuracy_reward/std": 0.3759314715862274, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.50537109375, + "rewards/tag_count_reward/std": 0.24846979975700378, + "step": 35 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.392578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2041.0, + "completions/mean_length": 1420.86328125, + "completions/mean_terminated_length": 1015.5433959960938, + "completions/min_length": 121.0, + "completions/min_terminated_length": 121.0, + "epoch": 0.01228983528206879, + "grad_norm": 0.12051548808813095, + "kl": 0.003536224365234375, + "learning_rate": 1.1986301369863011e-07, + "loss": 0.1341, + "num_tokens": 32216230.0, + "reward": 0.5908203125, + "reward_std": 0.3006790280342102, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.5224609375, + "rewards/tag_count_reward/std": 0.26768815517425537, + "step": 36 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.404296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 1517.08203125, + "completions/mean_terminated_length": 1156.754150390625, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.012631219595459589, + "grad_norm": 0.10653501749038696, + "kl": 0.00284576416015625, + "learning_rate": 1.232876712328767e-07, + "loss": 0.1508, + "num_tokens": 33067920.0, + "reward": 0.5849609375, + "reward_std": 0.3052826523780823, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.5087890625, + "rewards/tag_count_reward/std": 0.257796049118042, + "step": 37 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.392578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2009.0, + "completions/mean_length": 1439.75390625, + "completions/mean_terminated_length": 1046.64306640625, + "completions/min_length": 155.0, + "completions/min_terminated_length": 155.0, + "epoch": 0.012972603908850388, + "grad_norm": 0.12568874657154083, + "kl": 0.00372314453125, + "learning_rate": 1.2671232876712328e-07, + "loss": 0.1542, + "num_tokens": 33885794.0, + "reward": 0.57177734375, + "reward_std": 0.2767326831817627, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.51708984375, + "rewards/tag_count_reward/std": 0.2599788010120392, + "step": 38 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.35546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 1402.21875, + "completions/mean_terminated_length": 1046.060546875, + "completions/min_length": 181.0, + "completions/min_terminated_length": 181.0, + "epoch": 0.013313988222241188, + "grad_norm": 0.12105520069599152, + "kl": 0.00417327880859375, + "learning_rate": 1.3013698630136985e-07, + "loss": 0.1481, + "num_tokens": 34681634.0, + "reward": 0.6513671875, + "reward_std": 0.31875500082969666, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.5537109375, + "rewards/tag_count_reward/std": 0.27545711398124695, + "step": 39 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.5, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 1559.27734375, + "completions/mean_terminated_length": 1070.5546875, + "completions/min_length": 186.0, + "completions/min_terminated_length": 186.0, + "epoch": 0.013655372535631987, + "grad_norm": 0.10639850795269012, + "kl": 0.003894805908203125, + "learning_rate": 1.3356164383561644e-07, + "loss": 0.1527, + "num_tokens": 35560576.0, + "reward": 0.51171875, + "reward_std": 0.27677735686302185, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.46875, + "rewards/tag_count_reward/std": 0.2612442076206207, + "step": 40 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 1423.3515625, + "completions/mean_terminated_length": 1022.9359130859375, + "completions/min_length": 193.0, + "completions/min_terminated_length": 193.0, + "epoch": 0.013996756849022787, + "grad_norm": 0.117813341319561, + "kl": 0.00411224365234375, + "learning_rate": 1.36986301369863e-07, + "loss": 0.1249, + "num_tokens": 36360196.0, + "reward": 0.6103515625, + "reward_std": 0.2959108352661133, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.5380859375, + "rewards/tag_count_reward/std": 0.27805349230766296, + "step": 41 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.423828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 1436.888671875, + "completions/mean_terminated_length": 987.3593139648438, + "completions/min_length": 40.0, + "completions/min_terminated_length": 40.0, + "epoch": 0.014338141162413586, + "grad_norm": 0.13158966600894928, + "kl": 0.0052337646484375, + "learning_rate": 1.4041095890410958e-07, + "loss": 0.1634, + "num_tokens": 37181243.0, + "reward": 0.6552734375, + "reward_std": 0.3702911138534546, + "rewards/accuracy_reward/mean": 0.12109375, + "rewards/accuracy_reward/std": 0.3265552520751953, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.5341796875, + "rewards/tag_count_reward/std": 0.2790003716945648, + "step": 42 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.416015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 1464.73046875, + "completions/mean_terminated_length": 1049.22412109375, + "completions/min_length": 17.0, + "completions/min_terminated_length": 17.0, + "epoch": 0.014679525475804386, + "grad_norm": 0.11569352447986603, + "kl": 0.00490570068359375, + "learning_rate": 1.4383561643835615e-07, + "loss": 0.1364, + "num_tokens": 38004785.0, + "reward": 0.65087890625, + "reward_std": 0.3135306239128113, + "rewards/accuracy_reward/mean": 0.11328125, + "rewards/accuracy_reward/std": 0.3172462284564972, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.53759765625, + "rewards/tag_count_reward/std": 0.26940807700157166, + "step": 43 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.318359375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 1345.845703125, + "completions/mean_terminated_length": 1017.9054565429688, + "completions/min_length": 4.0, + "completions/min_terminated_length": 4.0, + "epoch": 0.015020909789195187, + "grad_norm": 0.11853042989969254, + "kl": 0.0054168701171875, + "learning_rate": 1.4726027397260274e-07, + "loss": 0.1455, + "num_tokens": 38764066.0, + "reward": 0.72412109375, + "reward_std": 0.38516783714294434, + "rewards/accuracy_reward/mean": 0.162109375, + "rewards/accuracy_reward/std": 0.3689115643501282, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.56201171875, + "rewards/tag_count_reward/std": 0.27570804953575134, + "step": 44 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 1304.82421875, + "completions/mean_terminated_length": 1014.0162963867188, + "completions/min_length": 41.0, + "completions/min_terminated_length": 41.0, + "epoch": 0.015362294102585987, + "grad_norm": 0.11778295040130615, + "kl": 0.00571441650390625, + "learning_rate": 1.506849315068493e-07, + "loss": 0.1641, + "num_tokens": 39513896.0, + "reward": 0.6630859375, + "reward_std": 0.2909233570098877, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.6005859375, + "rewards/tag_count_reward/std": 0.274289071559906, + "step": 45 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 1399.044921875, + "completions/mean_terminated_length": 1053.1947021484375, + "completions/min_length": 187.0, + "completions/min_terminated_length": 187.0, + "epoch": 0.015703678415976786, + "grad_norm": 0.12609730660915375, + "kl": 0.0064697265625, + "learning_rate": 1.5410958904109588e-07, + "loss": 0.1506, + "num_tokens": 40304655.0, + "reward": 0.64892578125, + "reward_std": 0.3167800009250641, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.55126953125, + "rewards/tag_count_reward/std": 0.26111936569213867, + "step": 46 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.287109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 1281.595703125, + "completions/mean_terminated_length": 972.9342651367188, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "epoch": 0.016045062729367586, + "grad_norm": 0.1254846602678299, + "kl": 0.00629425048828125, + "learning_rate": 1.5753424657534245e-07, + "loss": 0.1563, + "num_tokens": 41036672.0, + "reward": 0.69189453125, + "reward_std": 0.2690476179122925, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.61767578125, + "rewards/tag_count_reward/std": 0.28142958879470825, + "step": 47 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.298828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 1328.29296875, + "completions/mean_terminated_length": 1021.5654296875, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "epoch": 0.016386447042758386, + "grad_norm": 0.1133851557970047, + "kl": 0.0064239501953125, + "learning_rate": 1.6095890410958904e-07, + "loss": 0.1806, + "num_tokens": 41797462.0, + "reward": 0.67578125, + "reward_std": 0.3162664771080017, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.611328125, + "rewards/tag_count_reward/std": 0.2759232819080353, + "step": 48 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.29296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 1305.4296875, + "completions/mean_terminated_length": 997.73486328125, + "completions/min_length": 113.0, + "completions/min_terminated_length": 113.0, + "epoch": 0.016727831356149185, + "grad_norm": 0.14507456123828888, + "kl": 0.0088348388671875, + "learning_rate": 1.643835616438356e-07, + "loss": 0.1383, + "num_tokens": 42539234.0, + "reward": 0.70068359375, + "reward_std": 0.3099219799041748, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.61865234375, + "rewards/tag_count_reward/std": 0.2870470881462097, + "step": 49 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.24609375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 1211.787109375, + "completions/mean_terminated_length": 938.826416015625, + "completions/min_length": 37.0, + "completions/min_terminated_length": 37.0, + "epoch": 0.017069215669539985, + "grad_norm": 0.160403773188591, + "kl": 0.010650634765625, + "learning_rate": 1.6780821917808218e-07, + "loss": 0.1423, + "num_tokens": 43227733.0, + "reward": 0.76904296875, + "reward_std": 0.36772122979164124, + "rewards/accuracy_reward/mean": 0.126953125, + "rewards/accuracy_reward/std": 0.33324605226516724, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.64208984375, + "rewards/tag_count_reward/std": 0.2827405035495758, + "step": 50 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.26953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2030.0, + "completions/mean_length": 1242.466796875, + "completions/mean_terminated_length": 945.2379760742188, + "completions/min_length": 19.0, + "completions/min_terminated_length": 19.0, + "epoch": 0.017410599982930784, + "grad_norm": 0.13634954392910004, + "kl": 0.0076904296875, + "learning_rate": 1.7123287671232875e-07, + "loss": 0.1621, + "num_tokens": 43943108.0, + "reward": 0.79931640625, + "reward_std": 0.36965513229370117, + "rewards/accuracy_reward/mean": 0.1640625, + "rewards/accuracy_reward/std": 0.37069445848464966, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.63525390625, + "rewards/tag_count_reward/std": 0.28863388299942017, + "step": 51 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.232421875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 1197.734375, + "completions/mean_terminated_length": 940.2748413085938, + "completions/min_length": 4.0, + "completions/min_terminated_length": 4.0, + "epoch": 0.017751984296321584, + "grad_norm": 0.134590283036232, + "kl": 0.0091094970703125, + "learning_rate": 1.7465753424657535e-07, + "loss": 0.1779, + "num_tokens": 44634668.0, + "reward": 0.69775390625, + "reward_std": 0.2980614900588989, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.63330078125, + "rewards/tag_count_reward/std": 0.27031806111335754, + "step": 52 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.271484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 1235.2265625, + "completions/mean_terminated_length": 932.3432006835938, + "completions/min_length": 24.0, + "completions/min_terminated_length": 24.0, + "epoch": 0.018093368609712383, + "grad_norm": 0.4131734073162079, + "kl": 0.011260986328125, + "learning_rate": 1.780821917808219e-07, + "loss": 0.1614, + "num_tokens": 45341264.0, + "reward": 0.759765625, + "reward_std": 0.35761427879333496, + "rewards/accuracy_reward/mean": 0.134765625, + "rewards/accuracy_reward/std": 0.3418070077896118, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.625, + "rewards/tag_count_reward/std": 0.2789061665534973, + "step": 53 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.248046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 1147.494140625, + "completions/mean_terminated_length": 850.4441528320312, + "completions/min_length": 25.0, + "completions/min_terminated_length": 25.0, + "epoch": 0.018434752923103183, + "grad_norm": 0.14847879111766815, + "kl": 0.0095977783203125, + "learning_rate": 1.8150684931506848e-07, + "loss": 0.1749, + "num_tokens": 46011789.0, + "reward": 0.76416015625, + "reward_std": 0.3492337465286255, + "rewards/accuracy_reward/mean": 0.1171875, + "rewards/accuracy_reward/std": 0.32195815443992615, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.64697265625, + "rewards/tag_count_reward/std": 0.28541797399520874, + "step": 54 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 1239.833984375, + "completions/mean_terminated_length": 970.4453125, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "epoch": 0.018776137236493982, + "grad_norm": 0.12348626554012299, + "kl": 0.006805419921875, + "learning_rate": 1.8493150684931505e-07, + "loss": 0.1739, + "num_tokens": 46719448.0, + "reward": 0.73828125, + "reward_std": 0.35028383135795593, + "rewards/accuracy_reward/mean": 0.107421875, + "rewards/accuracy_reward/std": 0.30995169281959534, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.630859375, + "rewards/tag_count_reward/std": 0.27752548456192017, + "step": 55 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.150390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 1010.849609375, + "completions/mean_terminated_length": 827.2620849609375, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "epoch": 0.019117521549884782, + "grad_norm": 0.14299838244915009, + "kl": 0.00946044921875, + "learning_rate": 1.8835616438356165e-07, + "loss": 0.1366, + "num_tokens": 47314203.0, + "reward": 0.8544921875, + "reward_std": 0.3641713857650757, + "rewards/accuracy_reward/mean": 0.15234375, + "rewards/accuracy_reward/std": 0.35970520973205566, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.7021484375, + "rewards/tag_count_reward/std": 0.26891329884529114, + "step": 56 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.26171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2041.0, + "completions/mean_length": 1245.9609375, + "completions/mean_terminated_length": 961.6401977539062, + "completions/min_length": 64.0, + "completions/min_terminated_length": 64.0, + "epoch": 0.01945890586327558, + "grad_norm": 0.14915452897548676, + "kl": 0.00835418701171875, + "learning_rate": 1.917808219178082e-07, + "loss": 0.2204, + "num_tokens": 48030055.0, + "reward": 0.7587890625, + "reward_std": 0.3553071618080139, + "rewards/accuracy_reward/mean": 0.1015625, + "rewards/accuracy_reward/std": 0.30236753821372986, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.6572265625, + "rewards/tag_count_reward/std": 0.2924967110157013, + "step": 57 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.314453125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 1291.556640625, + "completions/mean_terminated_length": 944.5840454101562, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "epoch": 0.01980029017666638, + "grad_norm": 0.12248866260051727, + "kl": 0.00811004638671875, + "learning_rate": 1.9520547945205478e-07, + "loss": 0.2033, + "num_tokens": 48783668.0, + "reward": 0.697265625, + "reward_std": 0.3434180021286011, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.607421875, + "rewards/tag_count_reward/std": 0.2916542887687683, + "step": 58 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.28515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 1229.19140625, + "completions/mean_terminated_length": 902.5628051757812, + "completions/min_length": 170.0, + "completions/min_terminated_length": 170.0, + "epoch": 0.02014167449005718, + "grad_norm": 0.12459442764520645, + "kl": 0.00914764404296875, + "learning_rate": 1.9863013698630135e-07, + "loss": 0.2028, + "num_tokens": 49481446.0, + "reward": 0.74169921875, + "reward_std": 0.37255799770355225, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.64599609375, + "rewards/tag_count_reward/std": 0.2980671226978302, + "step": 59 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.275390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 1194.38671875, + "completions/mean_terminated_length": 869.9676513671875, + "completions/min_length": 133.0, + "completions/min_terminated_length": 133.0, + "epoch": 0.02048305880344798, + "grad_norm": 0.1351117342710495, + "kl": 0.0087432861328125, + "learning_rate": 2.0205479452054795e-07, + "loss": 0.1894, + "num_tokens": 50176396.0, + "reward": 0.73193359375, + "reward_std": 0.34207260608673096, + "rewards/accuracy_reward/mean": 0.087890625, + "rewards/accuracy_reward/std": 0.2834126651287079, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.64404296875, + "rewards/tag_count_reward/std": 0.2973770797252655, + "step": 60 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.33203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 1349.58984375, + "completions/mean_terminated_length": 1002.4268798828125, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "epoch": 0.02082444311683878, + "grad_norm": 0.11476152390241623, + "kl": 0.00765228271484375, + "learning_rate": 2.054794520547945e-07, + "loss": 0.1738, + "num_tokens": 50941514.0, + "reward": 0.73388671875, + "reward_std": 0.3434625267982483, + "rewards/accuracy_reward/mean": 0.1328125, + "rewards/accuracy_reward/std": 0.33970388770103455, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.60107421875, + "rewards/tag_count_reward/std": 0.2890937030315399, + "step": 61 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 1068.947265625, + "completions/mean_terminated_length": 831.3131103515625, + "completions/min_length": 118.0, + "completions/min_terminated_length": 118.0, + "epoch": 0.02116582743022958, + "grad_norm": 0.14407098293304443, + "kl": 0.0102691650390625, + "learning_rate": 2.0890410958904109e-07, + "loss": 0.1951, + "num_tokens": 51562751.0, + "reward": 0.8681640625, + "reward_std": 0.33195409178733826, + "rewards/accuracy_reward/mean": 0.171875, + "rewards/accuracy_reward/std": 0.3776407241821289, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.6962890625, + "rewards/tag_count_reward/std": 0.28029879927635193, + "step": 62 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.19140625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1995.0, + "completions/mean_length": 1083.150390625, + "completions/mean_terminated_length": 854.7560424804688, + "completions/min_length": 25.0, + "completions/min_terminated_length": 25.0, + "epoch": 0.021507211743620382, + "grad_norm": 0.159409761428833, + "kl": 0.0135955810546875, + "learning_rate": 2.1232876712328765e-07, + "loss": 0.1606, + "num_tokens": 52200380.0, + "reward": 0.82275390625, + "reward_std": 0.3297857344150543, + "rewards/accuracy_reward/mean": 0.1328125, + "rewards/accuracy_reward/std": 0.33970388770103455, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.68994140625, + "rewards/tag_count_reward/std": 0.27702537178993225, + "step": 63 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.154296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2001.0, + "completions/mean_length": 1013.60546875, + "completions/mean_terminated_length": 824.8822021484375, + "completions/min_length": 63.0, + "completions/min_terminated_length": 63.0, + "epoch": 0.021848596057011182, + "grad_norm": 0.16638486087322235, + "kl": 0.0105438232421875, + "learning_rate": 2.1575342465753425e-07, + "loss": 0.1404, + "num_tokens": 52798274.0, + "reward": 0.91357421875, + "reward_std": 0.3621740937232971, + "rewards/accuracy_reward/mean": 0.189453125, + "rewards/accuracy_reward/std": 0.3922513723373413, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.72412109375, + "rewards/tag_count_reward/std": 0.27168965339660645, + "step": 64 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 1173.52734375, + "completions/mean_terminated_length": 928.6749877929688, + "completions/min_length": 40.0, + "completions/min_terminated_length": 40.0, + "epoch": 0.02218998037040198, + "grad_norm": 0.12891757488250732, + "kl": 0.0115509033203125, + "learning_rate": 2.191780821917808e-07, + "loss": 0.1843, + "num_tokens": 53479248.0, + "reward": 0.8671875, + "reward_std": 0.35664865374565125, + "rewards/accuracy_reward/mean": 0.18359375, + "rewards/accuracy_reward/std": 0.3875311613082886, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.68359375, + "rewards/tag_count_reward/std": 0.29382818937301636, + "step": 65 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.271484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 1246.38671875, + "completions/mean_terminated_length": 947.6622314453125, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "epoch": 0.02253136468379278, + "grad_norm": 0.12625646591186523, + "kl": 0.00994873046875, + "learning_rate": 2.2260273972602739e-07, + "loss": 0.1674, + "num_tokens": 54194758.0, + "reward": 0.73876953125, + "reward_std": 0.2965083122253418, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.66259765625, + "rewards/tag_count_reward/std": 0.28932827711105347, + "step": 66 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.251953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 1216.79296875, + "completions/mean_terminated_length": 936.830322265625, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "epoch": 0.02287274899718358, + "grad_norm": 0.12288202345371246, + "kl": 0.0096435546875, + "learning_rate": 2.2602739726027396e-07, + "loss": 0.1781, + "num_tokens": 54886236.0, + "reward": 0.8212890625, + "reward_std": 0.3608725070953369, + "rewards/accuracy_reward/mean": 0.126953125, + "rewards/accuracy_reward/std": 0.33324605226516724, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.6943359375, + "rewards/tag_count_reward/std": 0.2993423044681549, + "step": 67 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 1220.8671875, + "completions/mean_terminated_length": 909.5806274414062, + "completions/min_length": 107.0, + "completions/min_terminated_length": 107.0, + "epoch": 0.02321413331057438, + "grad_norm": 0.2134619951248169, + "kl": 0.0125885009765625, + "learning_rate": 2.2945205479452055e-07, + "loss": 0.1652, + "num_tokens": 55588616.0, + "reward": 0.85107421875, + "reward_std": 0.3770219385623932, + "rewards/accuracy_reward/mean": 0.173828125, + "rewards/accuracy_reward/std": 0.3793322443962097, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.67724609375, + "rewards/tag_count_reward/std": 0.30800607800483704, + "step": 68 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.33203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 1318.947265625, + "completions/mean_terminated_length": 956.5526123046875, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "epoch": 0.02355551762396518, + "grad_norm": 0.13541601598262787, + "kl": 0.0112457275390625, + "learning_rate": 2.328767123287671e-07, + "loss": 0.2252, + "num_tokens": 56348333.0, + "reward": 0.66455078125, + "reward_std": 0.2940008044242859, + "rewards/accuracy_reward/mean": 0.018145160749554634, + "rewards/accuracy_reward/std": 0.1336110383272171, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.64697265625, + "rewards/tag_count_reward/std": 0.3124231696128845, + "step": 69 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.240234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2017.0, + "completions/mean_length": 1215.466796875, + "completions/mean_terminated_length": 952.2236328125, + "completions/min_length": 183.0, + "completions/min_terminated_length": 183.0, + "epoch": 0.02389690193735598, + "grad_norm": 0.12335766851902008, + "kl": 0.0104522705078125, + "learning_rate": 2.363013698630137e-07, + "loss": 0.206, + "num_tokens": 57049292.0, + "reward": 0.82177734375, + "reward_std": 0.35854047536849976, + "rewards/accuracy_reward/mean": 0.125, + "rewards/accuracy_reward/std": 0.3310423493385315, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.69677734375, + "rewards/tag_count_reward/std": 0.2987651824951172, + "step": 70 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2017.0, + "completions/mean_length": 1196.556640625, + "completions/mean_terminated_length": 906.7984619140625, + "completions/min_length": 98.0, + "completions/min_terminated_length": 98.0, + "epoch": 0.02423828625074678, + "grad_norm": 0.7646471261978149, + "kl": 0.0182647705078125, + "learning_rate": 2.3972602739726023e-07, + "loss": 0.2167, + "num_tokens": 57738057.0, + "reward": 0.7978515625, + "reward_std": 0.3488779664039612, + "rewards/accuracy_reward/mean": 0.109375, + "rewards/accuracy_reward/std": 0.31241437792778015, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.6884765625, + "rewards/tag_count_reward/std": 0.3090671896934509, + "step": 71 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.201171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 1155.95703125, + "completions/mean_terminated_length": 931.310546875, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "epoch": 0.02457967056413758, + "grad_norm": 0.13287417590618134, + "kl": 0.0117034912109375, + "learning_rate": 2.4315068493150685e-07, + "loss": 0.1464, + "num_tokens": 58416035.0, + "reward": 0.80859375, + "reward_std": 0.345120906829834, + "rewards/accuracy_reward/mean": 0.087890625, + "rewards/accuracy_reward/std": 0.2834126651287079, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.720703125, + "rewards/tag_count_reward/std": 0.2868976593017578, + "step": 72 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 1118.962890625, + "completions/mean_terminated_length": 870.6064453125, + "completions/min_length": 134.0, + "completions/min_terminated_length": 134.0, + "epoch": 0.024921054877528378, + "grad_norm": 0.13646827638149261, + "kl": 0.01141357421875, + "learning_rate": 2.465753424657534e-07, + "loss": 0.1898, + "num_tokens": 59075296.0, + "reward": 0.8046875, + "reward_std": 0.32949909567832947, + "rewards/accuracy_reward/mean": 0.099609375, + "rewards/accuracy_reward/std": 0.29977133870124817, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.705078125, + "rewards/tag_count_reward/std": 0.2908143401145935, + "step": 73 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.205078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2021.0, + "completions/mean_length": 1106.208984375, + "completions/mean_terminated_length": 863.2407836914062, + "completions/min_length": 146.0, + "completions/min_terminated_length": 146.0, + "epoch": 0.025262439190919177, + "grad_norm": 0.14528246223926544, + "kl": 0.011138916015625, + "learning_rate": 2.5e-07, + "loss": 0.1896, + "num_tokens": 59715547.0, + "reward": 0.91357421875, + "reward_std": 0.3325057923793793, + "rewards/accuracy_reward/mean": 0.181640625, + "rewards/accuracy_reward/std": 0.38592514395713806, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.73193359375, + "rewards/tag_count_reward/std": 0.28930845856666565, + "step": 74 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.240234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 1172.107421875, + "completions/mean_terminated_length": 895.1542358398438, + "completions/min_length": 147.0, + "completions/min_terminated_length": 147.0, + "epoch": 0.025603823504309977, + "grad_norm": 0.13012762367725372, + "kl": 0.01123046875, + "learning_rate": 2.5342465753424656e-07, + "loss": 0.2159, + "num_tokens": 60395666.0, + "reward": 0.7939453125, + "reward_std": 0.35236185789108276, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.7099609375, + "rewards/tag_count_reward/std": 0.30102303624153137, + "step": 75 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.173828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 1107.95703125, + "completions/mean_terminated_length": 910.1702270507812, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "epoch": 0.025945207817700777, + "grad_norm": 0.13782428205013275, + "kl": 0.0135498046875, + "learning_rate": 2.568493150684932e-07, + "loss": 0.1613, + "num_tokens": 61032620.0, + "reward": 0.92822265625, + "reward_std": 0.3581881821155548, + "rewards/accuracy_reward/mean": 0.173828125, + "rewards/accuracy_reward/std": 0.3793322443962097, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.75439453125, + "rewards/tag_count_reward/std": 0.28814682364463806, + "step": 76 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.216796875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 1145.453125, + "completions/mean_terminated_length": 895.6209716796875, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "epoch": 0.026286592131091576, + "grad_norm": 0.13462021946907043, + "kl": 0.012237548828125, + "learning_rate": 2.602739726027397e-07, + "loss": 0.2469, + "num_tokens": 61696596.0, + "reward": 0.8056640625, + "reward_std": 0.32687926292419434, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.7216796875, + "rewards/tag_count_reward/std": 0.29828062653541565, + "step": 77 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.18359375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 1071.12109375, + "completions/mean_terminated_length": 851.440185546875, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "epoch": 0.026627976444482376, + "grad_norm": 0.14011922478675842, + "kl": 0.0136871337890625, + "learning_rate": 2.6369863013698626e-07, + "loss": 0.2106, + "num_tokens": 62326322.0, + "reward": 0.82958984375, + "reward_std": 0.3330492377281189, + "rewards/accuracy_reward/mean": 0.1088709682226181, + "rewards/accuracy_reward/std": 0.31179171800613403, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.72412109375, + "rewards/tag_count_reward/std": 0.29706525802612305, + "step": 78 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.251953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2041.0, + "completions/mean_length": 1237.875, + "completions/mean_terminated_length": 965.0130615234375, + "completions/min_length": 222.0, + "completions/min_terminated_length": 222.0, + "epoch": 0.026969360757873175, + "grad_norm": 0.12821093201637268, + "kl": 0.0119171142578125, + "learning_rate": 2.671232876712329e-07, + "loss": 0.2283, + "num_tokens": 63040738.0, + "reward": 0.8056640625, + "reward_std": 0.36292368173599243, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.7294921875, + "rewards/tag_count_reward/std": 0.3125229477882385, + "step": 79 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.189453125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2023.0, + "completions/mean_length": 1023.4453125, + "completions/mean_terminated_length": 783.9711303710938, + "completions/min_length": 126.0, + "completions/min_terminated_length": 126.0, + "epoch": 0.027310745071263975, + "grad_norm": 0.15886980295181274, + "kl": 0.014007568359375, + "learning_rate": 2.7054794520547945e-07, + "loss": 0.2187, + "num_tokens": 63649078.0, + "reward": 0.8125, + "reward_std": 0.34062469005584717, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.755859375, + "rewards/tag_count_reward/std": 0.2908669114112854, + "step": 80 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2001.0, + "completions/mean_length": 1013.123046875, + "completions/mean_terminated_length": 810.016357421875, + "completions/min_length": 131.0, + "completions/min_terminated_length": 131.0, + "epoch": 0.027652129384654774, + "grad_norm": 0.14282691478729248, + "kl": 0.01397705078125, + "learning_rate": 2.73972602739726e-07, + "loss": 0.1899, + "num_tokens": 64239637.0, + "reward": 0.9736328125, + "reward_std": 0.3688603937625885, + "rewards/accuracy_reward/mean": 0.185546875, + "rewards/accuracy_reward/std": 0.38912075757980347, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.7880859375, + "rewards/tag_count_reward/std": 0.2841450273990631, + "step": 81 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1998.0, + "completions/mean_length": 1196.462890625, + "completions/mean_terminated_length": 906.6727905273438, + "completions/min_length": 175.0, + "completions/min_terminated_length": 175.0, + "epoch": 0.027993513698045574, + "grad_norm": 0.1358703076839447, + "kl": 0.012176513671875, + "learning_rate": 2.773972602739726e-07, + "loss": 0.2331, + "num_tokens": 64931634.0, + "reward": 0.86865234375, + "reward_std": 0.40526849031448364, + "rewards/accuracy_reward/mean": 0.166015625, + "rewards/accuracy_reward/std": 0.3724585771560669, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.70263671875, + "rewards/tag_count_reward/std": 0.31017935276031494, + "step": 82 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14453125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1994.0, + "completions/mean_length": 996.0390625, + "completions/mean_terminated_length": 818.3104858398438, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.028334898011436373, + "grad_norm": 0.165467768907547, + "kl": 0.015838623046875, + "learning_rate": 2.8082191780821916e-07, + "loss": 0.2608, + "num_tokens": 65515750.0, + "reward": 0.8486328125, + "reward_std": 0.3327805995941162, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.7705078125, + "rewards/tag_count_reward/std": 0.2816455364227295, + "step": 83 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.23046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 1168.208984375, + "completions/mean_terminated_length": 904.71826171875, + "completions/min_length": 40.0, + "completions/min_terminated_length": 40.0, + "epoch": 0.028676282324827173, + "grad_norm": 0.1360129863023758, + "kl": 0.013458251953125, + "learning_rate": 2.842465753424658e-07, + "loss": 0.1568, + "num_tokens": 66188513.0, + "reward": 0.83447265625, + "reward_std": 0.3465424180030823, + "rewards/accuracy_reward/mean": 0.099609375, + "rewards/accuracy_reward/std": 0.29977133870124817, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.73486328125, + "rewards/tag_count_reward/std": 0.3083006739616394, + "step": 84 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 1158.1171875, + "completions/mean_terminated_length": 885.7040405273438, + "completions/min_length": 127.0, + "completions/min_terminated_length": 127.0, + "epoch": 0.029017666638217973, + "grad_norm": 0.1278849095106125, + "kl": 0.0155487060546875, + "learning_rate": 2.876712328767123e-07, + "loss": 0.2267, + "num_tokens": 66857309.0, + "reward": 0.86767578125, + "reward_std": 0.3450787365436554, + "rewards/accuracy_reward/mean": 0.140625, + "rewards/accuracy_reward/std": 0.3479743003845215, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.72705078125, + "rewards/tag_count_reward/std": 0.303414523601532, + "step": 85 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.263671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2007.0, + "completions/mean_length": 1200.580078125, + "completions/mean_terminated_length": 897.1273193359375, + "completions/min_length": 139.0, + "completions/min_terminated_length": 139.0, + "epoch": 0.029359050951608772, + "grad_norm": 0.13545668125152588, + "kl": 0.0130767822265625, + "learning_rate": 2.9109589041095887e-07, + "loss": 0.1782, + "num_tokens": 67557238.0, + "reward": 0.85205078125, + "reward_std": 0.3195267915725708, + "rewards/accuracy_reward/mean": 0.130859375, + "rewards/accuracy_reward/std": 0.33757632970809937, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.72119140625, + "rewards/tag_count_reward/std": 0.31166985630989075, + "step": 86 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2016.0, + "completions/mean_length": 964.841796875, + "completions/mean_terminated_length": 770.1727905273438, + "completions/min_length": 4.0, + "completions/min_terminated_length": 4.0, + "epoch": 0.02970043526499957, + "grad_norm": 0.1605350822210312, + "kl": 0.01654052734375, + "learning_rate": 2.945205479452055e-07, + "loss": 0.1598, + "num_tokens": 68130005.0, + "reward": 0.87548828125, + "reward_std": 0.33797964453697205, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.77001953125, + "rewards/tag_count_reward/std": 0.30119559168815613, + "step": 87 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.228515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 1131.421875, + "completions/mean_terminated_length": 859.9291381835938, + "completions/min_length": 121.0, + "completions/min_terminated_length": 121.0, + "epoch": 0.030041819578390375, + "grad_norm": 0.1418033391237259, + "kl": 0.0140380859375, + "learning_rate": 2.9794520547945206e-07, + "loss": 0.1696, + "num_tokens": 68796397.0, + "reward": 0.90576171875, + "reward_std": 0.36803168058395386, + "rewards/accuracy_reward/mean": 0.1640625, + "rewards/accuracy_reward/std": 0.37069445848464966, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.74169921875, + "rewards/tag_count_reward/std": 0.3073694407939911, + "step": 88 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.19921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2033.0, + "completions/mean_length": 1130.34375, + "completions/mean_terminated_length": 902.0487670898438, + "completions/min_length": 148.0, + "completions/min_terminated_length": 148.0, + "epoch": 0.030383203891781174, + "grad_norm": 0.14273065328598022, + "kl": 0.0142974853515625, + "learning_rate": 3.013698630136986e-07, + "loss": 0.2161, + "num_tokens": 69448781.0, + "reward": 0.859375, + "reward_std": 0.36156928539276123, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.763671875, + "rewards/tag_count_reward/std": 0.2910245656967163, + "step": 89 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.22265625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2013.0, + "completions/mean_length": 1153.517578125, + "completions/mean_terminated_length": 897.3090209960938, + "completions/min_length": 120.0, + "completions/min_terminated_length": 120.0, + "epoch": 0.030724588205171974, + "grad_norm": 0.1517585963010788, + "kl": 0.01446533203125, + "learning_rate": 3.047945205479452e-07, + "loss": 0.2051, + "num_tokens": 70114438.0, + "reward": 0.86328125, + "reward_std": 0.3575511574745178, + "rewards/accuracy_reward/mean": 0.109375, + "rewards/accuracy_reward/std": 0.31241437792778015, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.75390625, + "rewards/tag_count_reward/std": 0.29672771692276, + "step": 90 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2014.0, + "completions/mean_length": 1065.25390625, + "completions/mean_terminated_length": 826.7233276367188, + "completions/min_length": 155.0, + "completions/min_terminated_length": 155.0, + "epoch": 0.031065972518562773, + "grad_norm": 0.13828644156455994, + "kl": 0.0145416259765625, + "learning_rate": 3.0821917808219176e-07, + "loss": 0.2289, + "num_tokens": 70735144.0, + "reward": 0.884765625, + "reward_std": 0.335534006357193, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.779296875, + "rewards/tag_count_reward/std": 0.3006371259689331, + "step": 91 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.248046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 1210.0546875, + "completions/mean_terminated_length": 933.6415405273438, + "completions/min_length": 113.0, + "completions/min_terminated_length": 113.0, + "epoch": 0.03140735683195357, + "grad_norm": 0.13442397117614746, + "kl": 0.0135650634765625, + "learning_rate": 3.116438356164384e-07, + "loss": 0.1872, + "num_tokens": 71439316.0, + "reward": 0.80322265625, + "reward_std": 0.3439953923225403, + "rewards/accuracy_reward/mean": 0.08749999850988388, + "rewards/accuracy_reward/std": 0.2828611731529236, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.72119140625, + "rewards/tag_count_reward/std": 0.30811774730682373, + "step": 92 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.16015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 967.873046875, + "completions/mean_terminated_length": 761.8953247070312, + "completions/min_length": 21.0, + "completions/min_terminated_length": 21.0, + "epoch": 0.03174874114534437, + "grad_norm": 0.15893946588039398, + "kl": 0.0166015625, + "learning_rate": 3.150684931506849e-07, + "loss": 0.1858, + "num_tokens": 72004003.0, + "reward": 0.94580078125, + "reward_std": 0.3452272117137909, + "rewards/accuracy_reward/mean": 0.142578125, + "rewards/accuracy_reward/std": 0.3499840497970581, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.80322265625, + "rewards/tag_count_reward/std": 0.28364479541778564, + "step": 93 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2009.0, + "completions/mean_length": 1193.263671875, + "completions/mean_terminated_length": 896.3552856445312, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "epoch": 0.03209012545873517, + "grad_norm": 0.15196147561073303, + "kl": 0.014251708984375, + "learning_rate": 3.1849315068493147e-07, + "loss": 0.2466, + "num_tokens": 72692618.0, + "reward": 0.80078125, + "reward_std": 0.3593321740627289, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.732421875, + "rewards/tag_count_reward/std": 0.31465137004852295, + "step": 94 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.19140625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2041.0, + "completions/mean_length": 1134.75390625, + "completions/mean_terminated_length": 918.5748901367188, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "epoch": 0.03243150977212597, + "grad_norm": 0.1329055279493332, + "kl": 0.013458251953125, + "learning_rate": 3.219178082191781e-07, + "loss": 0.1661, + "num_tokens": 73355356.0, + "reward": 0.90966796875, + "reward_std": 0.38059353828430176, + "rewards/accuracy_reward/mean": 0.146484375, + "rewards/accuracy_reward/std": 0.35393697023391724, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.76318359375, + "rewards/tag_count_reward/std": 0.2945975065231323, + "step": 95 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 1084.638671875, + "completions/mean_terminated_length": 814.8974609375, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "epoch": 0.03277289408551677, + "grad_norm": 0.17101000249385834, + "kl": 0.0164794921875, + "learning_rate": 3.2534246575342466e-07, + "loss": 0.1787, + "num_tokens": 73980531.0, + "reward": 0.876953125, + "reward_std": 0.3465830683708191, + "rewards/accuracy_reward/mean": 0.125, + "rewards/accuracy_reward/std": 0.3310423493385315, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.751953125, + "rewards/tag_count_reward/std": 0.3080716133117676, + "step": 96 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2023.0, + "completions/mean_length": 1056.6796875, + "completions/mean_terminated_length": 803.990234375, + "completions/min_length": 24.0, + "completions/min_terminated_length": 24.0, + "epoch": 0.03311427839890757, + "grad_norm": 0.15752506256103516, + "kl": 0.01513671875, + "learning_rate": 3.287671232876712e-07, + "loss": 0.2682, + "num_tokens": 74597599.0, + "reward": 0.90771484375, + "reward_std": 0.3453283905982971, + "rewards/accuracy_reward/mean": 0.1328125, + "rewards/accuracy_reward/std": 0.33970388770103455, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.77490234375, + "rewards/tag_count_reward/std": 0.29838111996650696, + "step": 97 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.263671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 1179.80078125, + "completions/mean_terminated_length": 868.9071655273438, + "completions/min_length": 122.0, + "completions/min_terminated_length": 122.0, + "epoch": 0.03345566271229837, + "grad_norm": 0.1455349624156952, + "kl": 0.0147705078125, + "learning_rate": 3.321917808219178e-07, + "loss": 0.2405, + "num_tokens": 75282553.0, + "reward": 0.81298828125, + "reward_std": 0.32863861322402954, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.73876953125, + "rewards/tag_count_reward/std": 0.3127991259098053, + "step": 98 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.216796875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2006.0, + "completions/mean_length": 1102.716796875, + "completions/mean_terminated_length": 841.0548706054688, + "completions/min_length": 20.0, + "completions/min_terminated_length": 20.0, + "epoch": 0.03379704702568917, + "grad_norm": 0.1469789743423462, + "kl": 0.015625, + "learning_rate": 3.3561643835616436e-07, + "loss": 0.2115, + "num_tokens": 75924024.0, + "reward": 0.86279296875, + "reward_std": 0.34248465299606323, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.78271484375, + "rewards/tag_count_reward/std": 0.30007997155189514, + "step": 99 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.20703125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2018.0, + "completions/mean_length": 1135.515625, + "completions/mean_terminated_length": 897.28076171875, + "completions/min_length": 109.0, + "completions/min_terminated_length": 109.0, + "epoch": 0.03413843133907997, + "grad_norm": 0.12671534717082977, + "kl": 0.0143890380859375, + "learning_rate": 3.39041095890411e-07, + "loss": 0.19, + "num_tokens": 76582544.0, + "reward": 0.90283203125, + "reward_std": 0.3243113160133362, + "rewards/accuracy_reward/mean": 0.12109375, + "rewards/accuracy_reward/std": 0.3265552520751953, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.78173828125, + "rewards/tag_count_reward/std": 0.3034271001815796, + "step": 100 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 1069.42578125, + "completions/mean_terminated_length": 898.8485717773438, + "completions/min_length": 180.0, + "completions/min_terminated_length": 180.0, + "epoch": 0.03447981565247077, + "grad_norm": 0.14338794350624084, + "kl": 0.016845703125, + "learning_rate": 3.424657534246575e-07, + "loss": 0.2441, + "num_tokens": 77201354.0, + "reward": 0.91015625, + "reward_std": 0.35537150502204895, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.81640625, + "rewards/tag_count_reward/std": 0.280627578496933, + "step": 101 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.205078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 1139.138671875, + "completions/mean_terminated_length": 904.6658325195312, + "completions/min_length": 129.0, + "completions/min_terminated_length": 129.0, + "epoch": 0.03482119996586157, + "grad_norm": 0.13609696924686432, + "kl": 0.016632080078125, + "learning_rate": 3.4589041095890407e-07, + "loss": 0.2475, + "num_tokens": 77861777.0, + "reward": 0.87158203125, + "reward_std": 0.3285045623779297, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.80126953125, + "rewards/tag_count_reward/std": 0.28954946994781494, + "step": 102 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 1077.892578125, + "completions/mean_terminated_length": 818.5569458007812, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "epoch": 0.03516258427925237, + "grad_norm": 0.1515011191368103, + "kl": 0.017059326171875, + "learning_rate": 3.493150684931507e-07, + "loss": 0.1885, + "num_tokens": 78491514.0, + "reward": 0.88818359375, + "reward_std": 0.3469456434249878, + "rewards/accuracy_reward/mean": 0.10685484111309052, + "rewards/accuracy_reward/std": 0.3092404901981354, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.78466796875, + "rewards/tag_count_reward/std": 0.3002677857875824, + "step": 103 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.16796875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2041.0, + "completions/mean_length": 1041.28125, + "completions/mean_terminated_length": 838.0469360351562, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "epoch": 0.03550396859264317, + "grad_norm": 0.127638041973114, + "kl": 0.0147857666015625, + "learning_rate": 3.5273972602739726e-07, + "loss": 0.1355, + "num_tokens": 79106090.0, + "reward": 1.02099609375, + "reward_std": 0.39763882756233215, + "rewards/accuracy_reward/mean": 0.22782258689403534, + "rewards/accuracy_reward/std": 0.4198509752750397, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.80029296875, + "rewards/tag_count_reward/std": 0.28331783413887024, + "step": 104 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 1055.228515625, + "completions/mean_terminated_length": 860.385498046875, + "completions/min_length": 120.0, + "completions/min_terminated_length": 120.0, + "epoch": 0.03584535290603397, + "grad_norm": 0.24287424981594086, + "kl": 0.017822265625, + "learning_rate": 3.561643835616438e-07, + "loss": 0.1756, + "num_tokens": 79722751.0, + "reward": 0.94384765625, + "reward_std": 0.30647194385528564, + "rewards/accuracy_reward/mean": 0.12109375, + "rewards/accuracy_reward/std": 0.3265552520751953, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.82275390625, + "rewards/tag_count_reward/std": 0.27573922276496887, + "step": 105 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.19921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2031.0, + "completions/mean_length": 1090.884765625, + "completions/mean_terminated_length": 852.773193359375, + "completions/min_length": 187.0, + "completions/min_terminated_length": 187.0, + "epoch": 0.03618673721942477, + "grad_norm": 0.14498759806156158, + "kl": 0.01666259765625, + "learning_rate": 3.595890410958904e-07, + "loss": 0.1711, + "num_tokens": 80364804.0, + "reward": 0.982421875, + "reward_std": 0.372316837310791, + "rewards/accuracy_reward/mean": 0.18359375, + "rewards/accuracy_reward/std": 0.3875311613082886, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.798828125, + "rewards/tag_count_reward/std": 0.29018279910087585, + "step": 106 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.142578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 1004.091796875, + "completions/mean_terminated_length": 830.5034790039062, + "completions/min_length": 96.0, + "completions/min_terminated_length": 96.0, + "epoch": 0.036528121532815566, + "grad_norm": 0.1424538493156433, + "kl": 0.01788330078125, + "learning_rate": 3.6301369863013697e-07, + "loss": 0.1613, + "num_tokens": 80956115.0, + "reward": 0.974609375, + "reward_std": 0.3216649293899536, + "rewards/accuracy_reward/mean": 0.138671875, + "rewards/accuracy_reward/std": 0.34594178199768066, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8359375, + "rewards/tag_count_reward/std": 0.26942625641822815, + "step": 107 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.236328125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 1194.708984375, + "completions/mean_terminated_length": 930.6470336914062, + "completions/min_length": 140.0, + "completions/min_terminated_length": 140.0, + "epoch": 0.036869505846206366, + "grad_norm": 0.12126666307449341, + "kl": 0.016265869140625, + "learning_rate": 3.664383561643836e-07, + "loss": 0.2176, + "num_tokens": 81649806.0, + "reward": 0.8544921875, + "reward_std": 0.3638536334037781, + "rewards/accuracy_reward/mean": 0.103515625, + "rewards/accuracy_reward/std": 0.30492907762527466, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.7509765625, + "rewards/tag_count_reward/std": 0.30926498770713806, + "step": 108 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.220703125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2041.0, + "completions/mean_length": 1093.07421875, + "completions/mean_terminated_length": 822.631591796875, + "completions/min_length": 134.0, + "completions/min_terminated_length": 134.0, + "epoch": 0.037210890159597165, + "grad_norm": 0.13973434269428253, + "kl": 0.019317626953125, + "learning_rate": 3.698630136986301e-07, + "loss": 0.1959, + "num_tokens": 82281076.0, + "reward": 0.90283203125, + "reward_std": 0.33090001344680786, + "rewards/accuracy_reward/mean": 0.115234375, + "rewards/accuracy_reward/std": 0.3196168541908264, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.78759765625, + "rewards/tag_count_reward/std": 0.3059704005718231, + "step": 109 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.166015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 1078.703125, + "completions/mean_terminated_length": 885.751708984375, + "completions/min_length": 149.0, + "completions/min_terminated_length": 149.0, + "epoch": 0.037552274472987965, + "grad_norm": 0.13122792541980743, + "kl": 0.0200347900390625, + "learning_rate": 3.7328767123287667e-07, + "loss": 0.1871, + "num_tokens": 82908444.0, + "reward": 0.9423828125, + "reward_std": 0.3743685483932495, + "rewards/accuracy_reward/mean": 0.130859375, + "rewards/accuracy_reward/std": 0.33757632970809937, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8115234375, + "rewards/tag_count_reward/std": 0.28732866048812866, + "step": 110 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.205078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2007.0, + "completions/mean_length": 1085.521484375, + "completions/mean_terminated_length": 837.2161865234375, + "completions/min_length": 123.0, + "completions/min_terminated_length": 123.0, + "epoch": 0.037893658786378764, + "grad_norm": 0.1464829444885254, + "kl": 0.01904296875, + "learning_rate": 3.767123287671233e-07, + "loss": 0.2646, + "num_tokens": 83541111.0, + "reward": 0.896484375, + "reward_std": 0.37763655185699463, + "rewards/accuracy_reward/mean": 0.103515625, + "rewards/accuracy_reward/std": 0.30492907762527466, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.79296875, + "rewards/tag_count_reward/std": 0.29898616671562195, + "step": 111 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 953.876953125, + "completions/mean_terminated_length": 840.6918334960938, + "completions/min_length": 107.0, + "completions/min_terminated_length": 107.0, + "epoch": 0.038235043099769564, + "grad_norm": 0.15541662275791168, + "kl": 0.01910400390625, + "learning_rate": 3.8013698630136986e-07, + "loss": 0.1899, + "num_tokens": 84104424.0, + "reward": 1.0166015625, + "reward_std": 0.33080601692199707, + "rewards/accuracy_reward/mean": 0.1484375, + "rewards/accuracy_reward/std": 0.35588082671165466, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8681640625, + "rewards/tag_count_reward/std": 0.24320943653583527, + "step": 112 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.169921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 1035.65234375, + "completions/mean_terminated_length": 828.4187622070312, + "completions/min_length": 82.0, + "completions/min_terminated_length": 82.0, + "epoch": 0.038576427413160363, + "grad_norm": 0.16102302074432373, + "kl": 0.0198974609375, + "learning_rate": 3.835616438356164e-07, + "loss": 0.1851, + "num_tokens": 84708902.0, + "reward": 0.96240234375, + "reward_std": 0.33114585280418396, + "rewards/accuracy_reward/mean": 0.15120968222618103, + "rewards/accuracy_reward/std": 0.35861483216285706, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.81591796875, + "rewards/tag_count_reward/std": 0.28312888741493225, + "step": 113 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.126953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 996.826171875, + "completions/mean_terminated_length": 843.970947265625, + "completions/min_length": 23.0, + "completions/min_terminated_length": 23.0, + "epoch": 0.03891781172655116, + "grad_norm": 0.15508544445037842, + "kl": 0.017730712890625, + "learning_rate": 3.86986301369863e-07, + "loss": 0.1869, + "num_tokens": 85298157.0, + "reward": 0.970703125, + "reward_std": 0.29038703441619873, + "rewards/accuracy_reward/mean": 0.138671875, + "rewards/accuracy_reward/std": 0.34594178199768066, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.83203125, + "rewards/tag_count_reward/std": 0.27064353227615356, + "step": 114 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.173828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 1132.140625, + "completions/mean_terminated_length": 939.4420776367188, + "completions/min_length": 149.0, + "completions/min_terminated_length": 149.0, + "epoch": 0.03925919603994196, + "grad_norm": 0.13651324808597565, + "kl": 0.0171051025390625, + "learning_rate": 3.9041095890410957e-07, + "loss": 0.1944, + "num_tokens": 85946037.0, + "reward": 1.00927734375, + "reward_std": 0.38964664936065674, + "rewards/accuracy_reward/mean": 0.1953125, + "rewards/accuracy_reward/std": 0.3968288004398346, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.81396484375, + "rewards/tag_count_reward/std": 0.28271347284317017, + "step": 115 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.212890625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 1104.9609375, + "completions/mean_terminated_length": 849.895751953125, + "completions/min_length": 110.0, + "completions/min_terminated_length": 110.0, + "epoch": 0.03960058035333276, + "grad_norm": 0.5017273426055908, + "kl": 0.0185699462890625, + "learning_rate": 3.938356164383562e-07, + "loss": 0.2407, + "num_tokens": 86581969.0, + "reward": 0.92626953125, + "reward_std": 0.3624322712421417, + "rewards/accuracy_reward/mean": 0.12109375, + "rewards/accuracy_reward/std": 0.3265552520751953, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.80517578125, + "rewards/tag_count_reward/std": 0.3020762503147125, + "step": 116 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.115234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1993.0, + "completions/mean_length": 922.08984375, + "completions/mean_terminated_length": 775.4481201171875, + "completions/min_length": 26.0, + "completions/min_terminated_length": 26.0, + "epoch": 0.03994196466672356, + "grad_norm": 0.17425385117530823, + "kl": 0.019683837890625, + "learning_rate": 3.972602739726027e-07, + "loss": 0.2106, + "num_tokens": 87127343.0, + "reward": 0.97802734375, + "reward_std": 0.33765822649002075, + "rewards/accuracy_reward/mean": 0.119140625, + "rewards/accuracy_reward/std": 0.32427072525024414, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.85888671875, + "rewards/tag_count_reward/std": 0.25361213088035583, + "step": 117 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.18359375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2021.0, + "completions/mean_length": 1080.203125, + "completions/mean_terminated_length": 862.5645751953125, + "completions/min_length": 117.0, + "completions/min_terminated_length": 117.0, + "epoch": 0.04028334898011436, + "grad_norm": 0.15133266150951385, + "kl": 0.019561767578125, + "learning_rate": 4.006849315068493e-07, + "loss": 0.1964, + "num_tokens": 87759863.0, + "reward": 0.953125, + "reward_std": 0.3536233603954315, + "rewards/accuracy_reward/mean": 0.142578125, + "rewards/accuracy_reward/std": 0.3499840497970581, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.810546875, + "rewards/tag_count_reward/std": 0.294263631105423, + "step": 118 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.26953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2013.0, + "completions/mean_length": 1211.7890625, + "completions/mean_terminated_length": 903.2406616210938, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.04062473329350516, + "grad_norm": 0.1288234293460846, + "kl": 0.018951416015625, + "learning_rate": 4.041095890410959e-07, + "loss": 0.2069, + "num_tokens": 88465835.0, + "reward": 0.9287109375, + "reward_std": 0.4196006655693054, + "rewards/accuracy_reward/mean": 0.154296875, + "rewards/accuracy_reward/std": 0.36158639192581177, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.7744140625, + "rewards/tag_count_reward/std": 0.3075050413608551, + "step": 119 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 1017.51953125, + "completions/mean_terminated_length": 779.7163696289062, + "completions/min_length": 109.0, + "completions/min_terminated_length": 109.0, + "epoch": 0.04096611760689596, + "grad_norm": 0.1571938693523407, + "kl": 0.02044677734375, + "learning_rate": 4.0753424657534246e-07, + "loss": 0.258, + "num_tokens": 89057525.0, + "reward": 0.978515625, + "reward_std": 0.38398873805999756, + "rewards/accuracy_reward/mean": 0.17578125, + "rewards/accuracy_reward/std": 0.3810062110424042, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.802734375, + "rewards/tag_count_reward/std": 0.29369157552719116, + "step": 120 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2013.0, + "completions/mean_length": 1028.244140625, + "completions/mean_terminated_length": 816.5967407226562, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "epoch": 0.04130750192028676, + "grad_norm": 0.1524491012096405, + "kl": 0.020660400390625, + "learning_rate": 4.10958904109589e-07, + "loss": 0.2634, + "num_tokens": 89660834.0, + "reward": 0.94384765625, + "reward_std": 0.32741492986679077, + "rewards/accuracy_reward/mean": 0.1270161271095276, + "rewards/accuracy_reward/std": 0.3333272337913513, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.82080078125, + "rewards/tag_count_reward/std": 0.285394549369812, + "step": 121 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.197265625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 1129.1484375, + "completions/mean_terminated_length": 903.347900390625, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "epoch": 0.04164888623367756, + "grad_norm": 0.14967991411685944, + "kl": 0.0193634033203125, + "learning_rate": 4.143835616438356e-07, + "loss": 0.2399, + "num_tokens": 90312590.0, + "reward": 0.9462890625, + "reward_std": 0.3499157726764679, + "rewards/accuracy_reward/mean": 0.14453125, + "rewards/accuracy_reward/std": 0.35197147727012634, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8017578125, + "rewards/tag_count_reward/std": 0.2967647314071655, + "step": 122 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.16796875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 1017.66796875, + "completions/mean_terminated_length": 809.6666870117188, + "completions/min_length": 35.0, + "completions/min_terminated_length": 35.0, + "epoch": 0.04199027054706836, + "grad_norm": 0.18192212283611298, + "kl": 0.02178955078125, + "learning_rate": 4.1780821917808217e-07, + "loss": 0.1701, + "num_tokens": 90911492.0, + "reward": 0.9931640625, + "reward_std": 0.38202834129333496, + "rewards/accuracy_reward/mean": 0.16796875, + "rewards/accuracy_reward/std": 0.374204158782959, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8251953125, + "rewards/tag_count_reward/std": 0.2775168716907501, + "step": 123 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1328125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2028.0, + "completions/mean_length": 939.9609375, + "completions/mean_terminated_length": 770.2612915039062, + "completions/min_length": 43.0, + "completions/min_terminated_length": 43.0, + "epoch": 0.04233165486045916, + "grad_norm": 0.1643485575914383, + "kl": 0.022003173828125, + "learning_rate": 4.212328767123288e-07, + "loss": 0.1862, + "num_tokens": 91465520.0, + "reward": 1.048828125, + "reward_std": 0.3515176773071289, + "rewards/accuracy_reward/mean": 0.17578125, + "rewards/accuracy_reward/std": 0.3810062110424042, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.873046875, + "rewards/tag_count_reward/std": 0.2497476041316986, + "step": 124 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.154296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2031.0, + "completions/mean_length": 944.138671875, + "completions/mean_terminated_length": 742.7413330078125, + "completions/min_length": 148.0, + "completions/min_terminated_length": 148.0, + "epoch": 0.04267303917384996, + "grad_norm": 0.41532760858535767, + "kl": 0.027557373046875, + "learning_rate": 4.246575342465753e-07, + "loss": 0.2359, + "num_tokens": 92035719.0, + "reward": 0.9375, + "reward_std": 0.30874431133270264, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.841796875, + "rewards/tag_count_reward/std": 0.27380964159965515, + "step": 125 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.177734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 1018.701171875, + "completions/mean_terminated_length": 796.2161865234375, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "epoch": 0.043014423487240765, + "grad_norm": 0.3897092640399933, + "kl": 0.0355224609375, + "learning_rate": 4.280821917808219e-07, + "loss": 0.2392, + "num_tokens": 92637870.0, + "reward": 0.93115234375, + "reward_std": 0.3366728723049164, + "rewards/accuracy_reward/mean": 0.103515625, + "rewards/accuracy_reward/std": 0.30492907762527466, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.82763671875, + "rewards/tag_count_reward/std": 0.284898579120636, + "step": 126 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.150390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 987.669921875, + "completions/mean_terminated_length": 799.9793090820312, + "completions/min_length": 113.0, + "completions/min_terminated_length": 113.0, + "epoch": 0.043355807800631564, + "grad_norm": 0.133542001247406, + "kl": 0.0201416015625, + "learning_rate": 4.315068493150685e-07, + "loss": 0.2343, + "num_tokens": 93214229.0, + "reward": 0.93701171875, + "reward_std": 0.3180956542491913, + "rewards/accuracy_reward/mean": 0.099609375, + "rewards/accuracy_reward/std": 0.29977133870124817, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.83740234375, + "rewards/tag_count_reward/std": 0.2776799499988556, + "step": 127 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.158203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2005.0, + "completions/mean_length": 992.728515625, + "completions/mean_terminated_length": 794.406005859375, + "completions/min_length": 149.0, + "completions/min_terminated_length": 149.0, + "epoch": 0.043697192114022364, + "grad_norm": 0.155024453997612, + "kl": 0.02215576171875, + "learning_rate": 4.3493150684931507e-07, + "loss": 0.2559, + "num_tokens": 93803706.0, + "reward": 0.92724609375, + "reward_std": 0.3279414772987366, + "rewards/accuracy_reward/mean": 0.103515625, + "rewards/accuracy_reward/std": 0.30492907762527466, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.82373046875, + "rewards/tag_count_reward/std": 0.2820604145526886, + "step": 128 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.130859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 1000.693359375, + "completions/mean_terminated_length": 843.0089721679688, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "epoch": 0.04403857642741316, + "grad_norm": 0.14573651552200317, + "kl": 0.021942138671875, + "learning_rate": 4.383561643835616e-07, + "loss": 0.2046, + "num_tokens": 94396829.0, + "reward": 0.97998046875, + "reward_std": 0.33831119537353516, + "rewards/accuracy_reward/mean": 0.140625, + "rewards/accuracy_reward/std": 0.3479743003845215, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.83935546875, + "rewards/tag_count_reward/std": 0.27035340666770935, + "step": 129 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.126953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 1026.716796875, + "completions/mean_terminated_length": 878.2080688476562, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.04437996074080396, + "grad_norm": 0.13692845404148102, + "kl": 0.02105712890625, + "learning_rate": 4.417808219178082e-07, + "loss": 0.1291, + "num_tokens": 95002332.0, + "reward": 0.97998046875, + "reward_std": 0.3523958921432495, + "rewards/accuracy_reward/mean": 0.138671875, + "rewards/accuracy_reward/std": 0.34594178199768066, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.84130859375, + "rewards/tag_count_reward/std": 0.26512494683265686, + "step": 130 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2890625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 1252.416015625, + "completions/mean_terminated_length": 928.9368286132812, + "completions/min_length": 156.0, + "completions/min_terminated_length": 156.0, + "epoch": 0.04472134505419476, + "grad_norm": 0.12116413563489914, + "kl": 0.021484375, + "learning_rate": 4.4520547945205477e-07, + "loss": 0.1845, + "num_tokens": 95722913.0, + "reward": 0.84814453125, + "reward_std": 0.3764338493347168, + "rewards/accuracy_reward/mean": 0.12890625, + "rewards/accuracy_reward/std": 0.33542385697364807, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.71923828125, + "rewards/tag_count_reward/std": 0.32793310284614563, + "step": 131 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.255859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2041.0, + "completions/mean_length": 1201.01953125, + "completions/mean_terminated_length": 909.800537109375, + "completions/min_length": 110.0, + "completions/min_terminated_length": 110.0, + "epoch": 0.04506272936758556, + "grad_norm": 0.1327248513698578, + "kl": 0.02069091796875, + "learning_rate": 4.4863013698630134e-07, + "loss": 0.1875, + "num_tokens": 96425483.0, + "reward": 0.89013671875, + "reward_std": 0.3610924482345581, + "rewards/accuracy_reward/mean": 0.150390625, + "rewards/accuracy_reward/std": 0.35780346393585205, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.73974609375, + "rewards/tag_count_reward/std": 0.3116576075553894, + "step": 132 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.216796875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 1162.76171875, + "completions/mean_terminated_length": 917.720703125, + "completions/min_length": 108.0, + "completions/min_terminated_length": 108.0, + "epoch": 0.04540411368097636, + "grad_norm": 0.14228351414203644, + "kl": 0.021026611328125, + "learning_rate": 4.520547945205479e-07, + "loss": 0.1876, + "num_tokens": 97095233.0, + "reward": 0.861328125, + "reward_std": 0.33468449115753174, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.783203125, + "rewards/tag_count_reward/std": 0.3002299964427948, + "step": 133 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1988.0, + "completions/mean_length": 1092.31640625, + "completions/mean_terminated_length": 830.8109130859375, + "completions/min_length": 129.0, + "completions/min_terminated_length": 129.0, + "epoch": 0.04574549799436716, + "grad_norm": 0.14528445899486542, + "kl": 0.02288818359375, + "learning_rate": 4.554794520547945e-07, + "loss": 0.2004, + "num_tokens": 97728499.0, + "reward": 0.89990234375, + "reward_std": 0.3530845642089844, + "rewards/accuracy_reward/mean": 0.115234375, + "rewards/accuracy_reward/std": 0.3196168541908264, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.78466796875, + "rewards/tag_count_reward/std": 0.30431386828422546, + "step": 134 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.17578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 1047.08984375, + "completions/mean_terminated_length": 833.6256103515625, + "completions/min_length": 75.0, + "completions/min_terminated_length": 75.0, + "epoch": 0.04608688230775796, + "grad_norm": 0.1438317745923996, + "kl": 0.022674560546875, + "learning_rate": 4.589041095890411e-07, + "loss": 0.2098, + "num_tokens": 98339809.0, + "reward": 1.0126953125, + "reward_std": 0.39924299716949463, + "rewards/accuracy_reward/mean": 0.21875, + "rewards/accuracy_reward/std": 0.41380295157432556, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.7939453125, + "rewards/tag_count_reward/std": 0.2918033301830292, + "step": 135 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.169921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 1103.76953125, + "completions/mean_terminated_length": 910.47998046875, + "completions/min_length": 5.0, + "completions/min_terminated_length": 5.0, + "epoch": 0.04642826662114876, + "grad_norm": 0.13441050052642822, + "kl": 0.02557373046875, + "learning_rate": 4.6232876712328767e-07, + "loss": 0.2169, + "num_tokens": 98983547.0, + "reward": 0.9375, + "reward_std": 0.33622604608535767, + "rewards/accuracy_reward/mean": 0.1270161271095276, + "rewards/accuracy_reward/std": 0.33332720398902893, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.814453125, + "rewards/tag_count_reward/std": 0.2854016423225403, + "step": 136 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.248046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2017.0, + "completions/mean_length": 1172.1171875, + "completions/mean_terminated_length": 883.1895751953125, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "epoch": 0.04676965093453956, + "grad_norm": 0.1310551017522812, + "kl": 0.021697998046875, + "learning_rate": 4.657534246575342e-07, + "loss": 0.1996, + "num_tokens": 99669991.0, + "reward": 0.8798828125, + "reward_std": 0.346016526222229, + "rewards/accuracy_reward/mean": 0.123046875, + "rewards/accuracy_reward/std": 0.32881227135658264, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.7568359375, + "rewards/tag_count_reward/std": 0.3158440887928009, + "step": 137 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 1104.271484375, + "completions/mean_terminated_length": 846.0372924804688, + "completions/min_length": 63.0, + "completions/min_terminated_length": 63.0, + "epoch": 0.04711103524793036, + "grad_norm": 0.13153010606765747, + "kl": 0.0242919921875, + "learning_rate": 4.691780821917808e-07, + "loss": 0.2479, + "num_tokens": 100306738.0, + "reward": 0.8525390625, + "reward_std": 0.33619534969329834, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.7724609375, + "rewards/tag_count_reward/std": 0.3032495379447937, + "step": 138 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2265625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2025.0, + "completions/mean_length": 1173.05859375, + "completions/mean_terminated_length": 916.7626342773438, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "epoch": 0.04745241956132116, + "grad_norm": 0.15235306322574615, + "kl": 0.0244140625, + "learning_rate": 4.726027397260274e-07, + "loss": 0.2333, + "num_tokens": 100988768.0, + "reward": 0.89306640625, + "reward_std": 0.33702439069747925, + "rewards/accuracy_reward/mean": 0.115234375, + "rewards/accuracy_reward/std": 0.3196168541908264, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.77783203125, + "rewards/tag_count_reward/std": 0.30381107330322266, + "step": 139 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.19921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 1096.26953125, + "completions/mean_terminated_length": 859.49755859375, + "completions/min_length": 122.0, + "completions/min_terminated_length": 122.0, + "epoch": 0.04779380387471196, + "grad_norm": 0.14825449883937836, + "kl": 0.0213623046875, + "learning_rate": 4.7602739726027394e-07, + "loss": 0.1917, + "num_tokens": 101621482.0, + "reward": 0.92822265625, + "reward_std": 0.3136889934539795, + "rewards/accuracy_reward/mean": 0.125, + "rewards/accuracy_reward/std": 0.3310528099536896, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.80712890625, + "rewards/tag_count_reward/std": 0.2897177040576935, + "step": 140 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 1127.798828125, + "completions/mean_terminated_length": 893.23779296875, + "completions/min_length": 112.0, + "completions/min_terminated_length": 112.0, + "epoch": 0.04813518818810276, + "grad_norm": 0.13674724102020264, + "kl": 0.0216064453125, + "learning_rate": 4.794520547945205e-07, + "loss": 0.2317, + "num_tokens": 102276419.0, + "reward": 0.9462890625, + "reward_std": 0.3898537755012512, + "rewards/accuracy_reward/mean": 0.154296875, + "rewards/accuracy_reward/std": 0.36158639192581177, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.7919921875, + "rewards/tag_count_reward/std": 0.2987160086631775, + "step": 141 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.23828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 1158.375, + "completions/mean_terminated_length": 880.0820922851562, + "completions/min_length": 156.0, + "completions/min_terminated_length": 156.0, + "epoch": 0.04847657250149356, + "grad_norm": 0.12955734133720398, + "kl": 0.023406982421875, + "learning_rate": 4.828767123287671e-07, + "loss": 0.2187, + "num_tokens": 102952787.0, + "reward": 0.88818359375, + "reward_std": 0.3411647081375122, + "rewards/accuracy_reward/mean": 0.107421875, + "rewards/accuracy_reward/std": 0.30995169281959534, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.78076171875, + "rewards/tag_count_reward/std": 0.3130495250225067, + "step": 142 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.23828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2020.0, + "completions/mean_length": 1182.376953125, + "completions/mean_terminated_length": 911.5923461914062, + "completions/min_length": 174.0, + "completions/min_terminated_length": 174.0, + "epoch": 0.04881795681488436, + "grad_norm": 0.1215982586145401, + "kl": 0.024078369140625, + "learning_rate": 4.863013698630137e-07, + "loss": 0.2179, + "num_tokens": 103631364.0, + "reward": 0.92236328125, + "reward_std": 0.3439157009124756, + "rewards/accuracy_reward/mean": 0.138671875, + "rewards/accuracy_reward/std": 0.34594178199768066, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.78369140625, + "rewards/tag_count_reward/std": 0.3123558759689331, + "step": 143 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.23046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 1130.21875, + "completions/mean_terminated_length": 855.3502197265625, + "completions/min_length": 163.0, + "completions/min_terminated_length": 163.0, + "epoch": 0.04915934112827516, + "grad_norm": 0.15443135797977448, + "kl": 0.0263671875, + "learning_rate": 4.897260273972603e-07, + "loss": 0.1898, + "num_tokens": 104289572.0, + "reward": 0.94482421875, + "reward_std": 0.36273717880249023, + "rewards/accuracy_reward/mean": 0.154296875, + "rewards/accuracy_reward/std": 0.36158639192581177, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.79052734375, + "rewards/tag_count_reward/std": 0.30479392409324646, + "step": 144 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.140625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2010.0, + "completions/mean_length": 935.13671875, + "completions/mean_terminated_length": 753.0317993164062, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "epoch": 0.049500725441665956, + "grad_norm": 0.16990253329277039, + "kl": 0.03143310546875, + "learning_rate": 4.931506849315068e-07, + "loss": 0.2292, + "num_tokens": 104838666.0, + "reward": 1.03076171875, + "reward_std": 0.35108163952827454, + "rewards/accuracy_reward/mean": 0.19921875, + "rewards/accuracy_reward/std": 0.39980348944664, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.83154296875, + "rewards/tag_count_reward/std": 0.26829564571380615, + "step": 145 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.197265625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2025.0, + "completions/mean_length": 1123.224609375, + "completions/mean_terminated_length": 895.9683837890625, + "completions/min_length": 139.0, + "completions/min_terminated_length": 139.0, + "epoch": 0.049842109755056756, + "grad_norm": 0.13389046490192413, + "kl": 0.026123046875, + "learning_rate": 4.965753424657534e-07, + "loss": 0.1889, + "num_tokens": 105487933.0, + "reward": 0.92431640625, + "reward_std": 0.3295518159866333, + "rewards/accuracy_reward/mean": 0.12890625, + "rewards/accuracy_reward/std": 0.33542385697364807, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.79541015625, + "rewards/tag_count_reward/std": 0.2959502041339874, + "step": 146 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.162109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 1083.4921875, + "completions/mean_terminated_length": 896.8858032226562, + "completions/min_length": 150.0, + "completions/min_terminated_length": 150.0, + "epoch": 0.050183494068447555, + "grad_norm": 0.12234027683734894, + "kl": 0.026885986328125, + "learning_rate": 5e-07, + "loss": 0.1988, + "num_tokens": 106121801.0, + "reward": 0.97607421875, + "reward_std": 0.3266531229019165, + "rewards/accuracy_reward/mean": 0.1391129046678543, + "rewards/accuracy_reward/std": 0.3464137017726898, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.84130859375, + "rewards/tag_count_reward/std": 0.27553123235702515, + "step": 147 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.228515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2028.0, + "completions/mean_length": 1198.45703125, + "completions/mean_terminated_length": 946.8202514648438, + "completions/min_length": 196.0, + "completions/min_terminated_length": 196.0, + "epoch": 0.050524878381838355, + "grad_norm": 0.12022490799427032, + "kl": 0.024627685546875, + "learning_rate": 5.034246575342465e-07, + "loss": 0.2133, + "num_tokens": 106814163.0, + "reward": 0.9658203125, + "reward_std": 0.3609035015106201, + "rewards/accuracy_reward/mean": 0.1796875, + "rewards/accuracy_reward/std": 0.38430243730545044, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.7861328125, + "rewards/tag_count_reward/std": 0.3063468933105469, + "step": 148 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.27734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2030.0, + "completions/mean_length": 1268.662109375, + "completions/mean_terminated_length": 969.5648803710938, + "completions/min_length": 169.0, + "completions/min_terminated_length": 169.0, + "epoch": 0.050866262695229154, + "grad_norm": 0.12187255173921585, + "kl": 0.02496337890625, + "learning_rate": 5.068493150684931e-07, + "loss": 0.2075, + "num_tokens": 107541798.0, + "reward": 0.8916015625, + "reward_std": 0.3265082538127899, + "rewards/accuracy_reward/mean": 0.126953125, + "rewards/accuracy_reward/std": 0.33324605226516724, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.7646484375, + "rewards/tag_count_reward/std": 0.3182791769504547, + "step": 149 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.20703125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2029.0, + "completions/mean_length": 1077.37890625, + "completions/mean_terminated_length": 823.9655151367188, + "completions/min_length": 146.0, + "completions/min_terminated_length": 146.0, + "epoch": 0.051207647008619954, + "grad_norm": 0.12971220910549164, + "kl": 0.03204345703125, + "learning_rate": 5.102739726027398e-07, + "loss": 0.2329, + "num_tokens": 108168616.0, + "reward": 0.9423828125, + "reward_std": 0.34843137860298157, + "rewards/accuracy_reward/mean": 0.119140625, + "rewards/accuracy_reward/std": 0.32427072525024414, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8232421875, + "rewards/tag_count_reward/std": 0.2913576364517212, + "step": 150 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.224609375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2031.0, + "completions/mean_length": 1141.125, + "completions/mean_terminated_length": 878.4281616210938, + "completions/min_length": 142.0, + "completions/min_terminated_length": 142.0, + "epoch": 0.051549031322010753, + "grad_norm": 0.1362224668264389, + "kl": 0.028289794921875, + "learning_rate": 5.136986301369864e-07, + "loss": 0.2354, + "num_tokens": 108833640.0, + "reward": 0.87890625, + "reward_std": 0.30723893642425537, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.802734375, + "rewards/tag_count_reward/std": 0.30231064558029175, + "step": 151 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.220703125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2000.0, + "completions/mean_length": 1187.841796875, + "completions/mean_terminated_length": 944.2380981445312, + "completions/min_length": 142.0, + "completions/min_terminated_length": 142.0, + "epoch": 0.05189041563540155, + "grad_norm": 0.13775372505187988, + "kl": 0.0277099609375, + "learning_rate": 5.171232876712328e-07, + "loss": 0.1825, + "num_tokens": 109526151.0, + "reward": 0.93359375, + "reward_std": 0.33710235357284546, + "rewards/accuracy_reward/mean": 0.12109375, + "rewards/accuracy_reward/std": 0.3265552520751953, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8125, + "rewards/tag_count_reward/std": 0.2942701280117035, + "step": 152 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.173828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2030.0, + "completions/mean_length": 1140.255859375, + "completions/mean_terminated_length": 949.2647705078125, + "completions/min_length": 158.0, + "completions/min_terminated_length": 158.0, + "epoch": 0.05223179994879235, + "grad_norm": 0.12133099883794785, + "kl": 0.025604248046875, + "learning_rate": 5.205479452054794e-07, + "loss": 0.214, + "num_tokens": 110184522.0, + "reward": 0.92578125, + "reward_std": 0.3521662950515747, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.828125, + "rewards/tag_count_reward/std": 0.28325754404067993, + "step": 153 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.177734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 1113.87890625, + "completions/mean_terminated_length": 911.966796875, + "completions/min_length": 117.0, + "completions/min_terminated_length": 117.0, + "epoch": 0.05257318426218315, + "grad_norm": 0.22379489243030548, + "kl": 0.02886962890625, + "learning_rate": 5.23972602739726e-07, + "loss": 0.1919, + "num_tokens": 110828988.0, + "reward": 0.99609375, + "reward_std": 0.3244497776031494, + "rewards/accuracy_reward/mean": 0.154296875, + "rewards/accuracy_reward/std": 0.36158639192581177, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.841796875, + "rewards/tag_count_reward/std": 0.2755906283855438, + "step": 154 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.166015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2031.0, + "completions/mean_length": 1084.51953125, + "completions/mean_terminated_length": 892.7259521484375, + "completions/min_length": 82.0, + "completions/min_terminated_length": 82.0, + "epoch": 0.05291456857557395, + "grad_norm": 0.13794507086277008, + "kl": 0.03118896484375, + "learning_rate": 5.273972602739725e-07, + "loss": 0.1815, + "num_tokens": 111451590.0, + "reward": 0.97216796875, + "reward_std": 0.33759379386901855, + "rewards/accuracy_reward/mean": 0.13306452333927155, + "rewards/accuracy_reward/std": 0.3399873375892639, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.84326171875, + "rewards/tag_count_reward/std": 0.27084073424339294, + "step": 155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.17578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2006.0, + "completions/mean_length": 1096.755859375, + "completions/mean_terminated_length": 893.8839111328125, + "completions/min_length": 140.0, + "completions/min_terminated_length": 140.0, + "epoch": 0.05325595288896475, + "grad_norm": 0.1420031189918518, + "kl": 0.032135009765625, + "learning_rate": 5.308219178082192e-07, + "loss": 0.221, + "num_tokens": 112086137.0, + "reward": 0.98486328125, + "reward_std": 0.36912697553634644, + "rewards/accuracy_reward/mean": 0.1484375, + "rewards/accuracy_reward/std": 0.35588082671165466, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.83642578125, + "rewards/tag_count_reward/std": 0.28148388862609863, + "step": 156 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.23046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 1106.5625, + "completions/mean_terminated_length": 824.609130859375, + "completions/min_length": 102.0, + "completions/min_terminated_length": 102.0, + "epoch": 0.05359733720235555, + "grad_norm": 0.16936340928077698, + "kl": 0.032470703125, + "learning_rate": 5.342465753424658e-07, + "loss": 0.2465, + "num_tokens": 112722537.0, + "reward": 0.96142578125, + "reward_std": 0.337054967880249, + "rewards/accuracy_reward/mean": 0.15625, + "rewards/accuracy_reward/std": 0.36344730854034424, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.80517578125, + "rewards/tag_count_reward/std": 0.29840993881225586, + "step": 157 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.19921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2008.0, + "completions/mean_length": 1127.84765625, + "completions/mean_terminated_length": 898.9317016601562, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "epoch": 0.05393872151574635, + "grad_norm": 0.13756844401359558, + "kl": 0.031005859375, + "learning_rate": 5.376712328767123e-07, + "loss": 0.2062, + "num_tokens": 113380459.0, + "reward": 0.89306640625, + "reward_std": 0.32475483417510986, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.80908203125, + "rewards/tag_count_reward/std": 0.2943541407585144, + "step": 158 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2028.0, + "completions/mean_length": 1088.658203125, + "completions/mean_terminated_length": 900.3761596679688, + "completions/min_length": 140.0, + "completions/min_terminated_length": 140.0, + "epoch": 0.05428010582913715, + "grad_norm": 0.1365545094013214, + "kl": 0.029693603515625, + "learning_rate": 5.410958904109589e-07, + "loss": 0.2414, + "num_tokens": 114018076.0, + "reward": 0.94287109375, + "reward_std": 0.3527100682258606, + "rewards/accuracy_reward/mean": 0.10685484111309052, + "rewards/accuracy_reward/std": 0.30924052000045776, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.83935546875, + "rewards/tag_count_reward/std": 0.2730543613433838, + "step": 159 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.19921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2002.0, + "completions/mean_length": 1097.521484375, + "completions/mean_terminated_length": 861.0609741210938, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "epoch": 0.05462149014252795, + "grad_norm": 0.14326536655426025, + "kl": 0.027130126953125, + "learning_rate": 5.445205479452054e-07, + "loss": 0.2094, + "num_tokens": 114656919.0, + "reward": 1.00439453125, + "reward_std": 0.3649257719516754, + "rewards/accuracy_reward/mean": 0.17578125, + "rewards/accuracy_reward/std": 0.3810062110424042, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.82861328125, + "rewards/tag_count_reward/std": 0.2901626229286194, + "step": 160 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.208984375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 1130.7265625, + "completions/mean_terminated_length": 888.38525390625, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.05496287445591875, + "grad_norm": 0.13187538087368011, + "kl": 0.03009033203125, + "learning_rate": 5.47945205479452e-07, + "loss": 0.1951, + "num_tokens": 115317627.0, + "reward": 0.9404296875, + "reward_std": 0.3498813509941101, + "rewards/accuracy_reward/mean": 0.126953125, + "rewards/accuracy_reward/std": 0.33324605226516724, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8134765625, + "rewards/tag_count_reward/std": 0.29613298177719116, + "step": 161 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.126953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 969.990234375, + "completions/mean_terminated_length": 813.232666015625, + "completions/min_length": 118.0, + "completions/min_terminated_length": 118.0, + "epoch": 0.05530425876930955, + "grad_norm": 0.14999741315841675, + "kl": 0.030303955078125, + "learning_rate": 5.513698630136986e-07, + "loss": 0.1619, + "num_tokens": 115890934.0, + "reward": 0.99169921875, + "reward_std": 0.30944162607192993, + "rewards/accuracy_reward/mean": 0.134765625, + "rewards/accuracy_reward/std": 0.3418070077896118, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.85693359375, + "rewards/tag_count_reward/std": 0.25873351097106934, + "step": 162 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 1132.5078125, + "completions/mean_terminated_length": 899.1470947265625, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "epoch": 0.05564564308270035, + "grad_norm": 0.1272873431444168, + "kl": 0.0289306640625, + "learning_rate": 5.547945205479452e-07, + "loss": 0.1552, + "num_tokens": 116547306.0, + "reward": 0.95068359375, + "reward_std": 0.338924765586853, + "rewards/accuracy_reward/mean": 0.1484375, + "rewards/accuracy_reward/std": 0.35588082671165466, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.80224609375, + "rewards/tag_count_reward/std": 0.2944028377532959, + "step": 163 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.17578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 1105.451171875, + "completions/mean_terminated_length": 904.4336547851562, + "completions/min_length": 108.0, + "completions/min_terminated_length": 108.0, + "epoch": 0.05598702739609115, + "grad_norm": 0.1345742791891098, + "kl": 0.028045654296875, + "learning_rate": 5.582191780821918e-07, + "loss": 0.1839, + "num_tokens": 117191233.0, + "reward": 0.99365234375, + "reward_std": 0.3265041708946228, + "rewards/accuracy_reward/mean": 0.1640625, + "rewards/accuracy_reward/std": 0.37069445848464966, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.82958984375, + "rewards/tag_count_reward/std": 0.2782711982727051, + "step": 164 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.224609375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 1171.6640625, + "completions/mean_terminated_length": 917.8135375976562, + "completions/min_length": 109.0, + "completions/min_terminated_length": 109.0, + "epoch": 0.05632841170948195, + "grad_norm": 0.13051313161849976, + "kl": 0.026824951171875, + "learning_rate": 5.616438356164383e-07, + "loss": 0.2358, + "num_tokens": 117867045.0, + "reward": 0.91064453125, + "reward_std": 0.34272146224975586, + "rewards/accuracy_reward/mean": 0.107421875, + "rewards/accuracy_reward/std": 0.30995169281959534, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.80322265625, + "rewards/tag_count_reward/std": 0.2983555197715759, + "step": 165 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.19140625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2030.0, + "completions/mean_length": 1097.896484375, + "completions/mean_terminated_length": 872.9927368164062, + "completions/min_length": 128.0, + "completions/min_terminated_length": 128.0, + "epoch": 0.05666979602287275, + "grad_norm": 0.12242324650287628, + "kl": 0.027252197265625, + "learning_rate": 5.65068493150685e-07, + "loss": 0.1292, + "num_tokens": 118506240.0, + "reward": 0.9541015625, + "reward_std": 0.33134326338768005, + "rewards/accuracy_reward/mean": 0.126953125, + "rewards/accuracy_reward/std": 0.33324605226516724, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8271484375, + "rewards/tag_count_reward/std": 0.2878071367740631, + "step": 166 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.150390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2023.0, + "completions/mean_length": 1049.802734375, + "completions/mean_terminated_length": 873.1103515625, + "completions/min_length": 178.0, + "completions/min_terminated_length": 178.0, + "epoch": 0.057011180336263546, + "grad_norm": 0.13246352970600128, + "kl": 0.030670166015625, + "learning_rate": 5.684931506849316e-07, + "loss": 0.2013, + "num_tokens": 119114347.0, + "reward": 1.01171875, + "reward_std": 0.3280482292175293, + "rewards/accuracy_reward/mean": 0.15234375, + "rewards/accuracy_reward/std": 0.35970520973205566, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.859375, + "rewards/tag_count_reward/std": 0.26588478684425354, + "step": 167 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.248046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2005.0, + "completions/mean_length": 1184.060546875, + "completions/mean_terminated_length": 899.0726928710938, + "completions/min_length": 145.0, + "completions/min_terminated_length": 145.0, + "epoch": 0.057352564649654346, + "grad_norm": 0.14424820244312286, + "kl": 0.0301513671875, + "learning_rate": 5.71917808219178e-07, + "loss": 0.2438, + "num_tokens": 119807194.0, + "reward": 0.85400390625, + "reward_std": 0.33242154121398926, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.78759765625, + "rewards/tag_count_reward/std": 0.30875593423843384, + "step": 168 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.18359375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 1022.5390625, + "completions/mean_terminated_length": 791.9329833984375, + "completions/min_length": 180.0, + "completions/min_terminated_length": 180.0, + "epoch": 0.057693948963045145, + "grad_norm": 0.1473248302936554, + "kl": 0.031097412109375, + "learning_rate": 5.753424657534246e-07, + "loss": 0.2123, + "num_tokens": 120404094.0, + "reward": 1.0205078125, + "reward_std": 0.38394251465797424, + "rewards/accuracy_reward/mean": 0.193359375, + "rewards/accuracy_reward/std": 0.39531853795051575, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8271484375, + "rewards/tag_count_reward/std": 0.28695595264434814, + "step": 169 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.20703125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 1166.875, + "completions/mean_terminated_length": 936.8275756835938, + "completions/min_length": 165.0, + "completions/min_terminated_length": 165.0, + "epoch": 0.058035333276435945, + "grad_norm": 0.13750946521759033, + "kl": 0.0291748046875, + "learning_rate": 5.787671232876712e-07, + "loss": 0.2577, + "num_tokens": 121080414.0, + "reward": 0.8798828125, + "reward_std": 0.33640050888061523, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8212890625, + "rewards/tag_count_reward/std": 0.2926796078681946, + "step": 170 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 1107.908203125, + "completions/mean_terminated_length": 890.9639892578125, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "epoch": 0.058376717589826745, + "grad_norm": 0.11815839260816574, + "kl": 0.029205322265625, + "learning_rate": 5.821917808219177e-07, + "loss": 0.1419, + "num_tokens": 121725167.0, + "reward": 0.94775390625, + "reward_std": 0.3215233087539673, + "rewards/accuracy_reward/mean": 0.111328125, + "rewards/accuracy_reward/std": 0.31484565138816833, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.83642578125, + "rewards/tag_count_reward/std": 0.2853677570819855, + "step": 171 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.185546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1985.0, + "completions/mean_length": 1102.84375, + "completions/mean_terminated_length": 887.5203857421875, + "completions/min_length": 130.0, + "completions/min_terminated_length": 130.0, + "epoch": 0.058718101903217544, + "grad_norm": 0.14123791456222534, + "kl": 0.0301513671875, + "learning_rate": 5.856164383561644e-07, + "loss": 0.1557, + "num_tokens": 122375455.0, + "reward": 0.93408203125, + "reward_std": 0.3450877368450165, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.82861328125, + "rewards/tag_count_reward/std": 0.27722880244255066, + "step": 172 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 1187.037109375, + "completions/mean_terminated_length": 956.8787231445312, + "completions/min_length": 83.0, + "completions/min_terminated_length": 83.0, + "epoch": 0.059059486216608344, + "grad_norm": 0.11834447830915451, + "kl": 0.028106689453125, + "learning_rate": 5.89041095890411e-07, + "loss": 0.1168, + "num_tokens": 123058930.0, + "reward": 0.9228515625, + "reward_std": 0.3344650864601135, + "rewards/accuracy_reward/mean": 0.099609375, + "rewards/accuracy_reward/std": 0.29977133870124817, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8232421875, + "rewards/tag_count_reward/std": 0.28755471110343933, + "step": 173 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2421875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 1200.091796875, + "completions/mean_terminated_length": 929.1107788085938, + "completions/min_length": 128.0, + "completions/min_terminated_length": 128.0, + "epoch": 0.05940087052999914, + "grad_norm": 0.13264034688472748, + "kl": 0.02886962890625, + "learning_rate": 5.924657534246575e-07, + "loss": 0.2176, + "num_tokens": 123753393.0, + "reward": 0.99560546875, + "reward_std": 0.39991408586502075, + "rewards/accuracy_reward/mean": 0.19959677755832672, + "rewards/accuracy_reward/std": 0.40010079741477966, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.80224609375, + "rewards/tag_count_reward/std": 0.29357075691223145, + "step": 174 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 1051.470703125, + "completions/mean_terminated_length": 914.171142578125, + "completions/min_length": 142.0, + "completions/min_terminated_length": 142.0, + "epoch": 0.05974225484338994, + "grad_norm": 0.13750015199184418, + "kl": 0.029754638671875, + "learning_rate": 5.958904109589041e-07, + "loss": 0.2035, + "num_tokens": 124369490.0, + "reward": 1.00341796875, + "reward_std": 0.35021504759788513, + "rewards/accuracy_reward/mean": 0.142578125, + "rewards/accuracy_reward/std": 0.3499840497970581, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.86083984375, + "rewards/tag_count_reward/std": 0.2556498050689697, + "step": 175 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1328125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2006.0, + "completions/mean_length": 994.48828125, + "completions/mean_terminated_length": 833.1396484375, + "completions/min_length": 147.0, + "completions/min_terminated_length": 147.0, + "epoch": 0.06008363915678075, + "grad_norm": 0.14909285306930542, + "kl": 0.029937744140625, + "learning_rate": 5.993150684931506e-07, + "loss": 0.1983, + "num_tokens": 124958524.0, + "reward": 0.98681640625, + "reward_std": 0.3174964189529419, + "rewards/accuracy_reward/mean": 0.12298387289047241, + "rewards/accuracy_reward/std": 0.32875028252601624, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.86767578125, + "rewards/tag_count_reward/std": 0.2515996992588043, + "step": 176 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 1125.064453125, + "completions/mean_terminated_length": 901.0509643554688, + "completions/min_length": 72.0, + "completions/min_terminated_length": 72.0, + "epoch": 0.06042502347017155, + "grad_norm": 0.1393154114484787, + "kl": 0.03125, + "learning_rate": 6.027397260273972e-07, + "loss": 0.2254, + "num_tokens": 125612029.0, + "reward": 0.96240234375, + "reward_std": 0.35647714138031006, + "rewards/accuracy_reward/mean": 0.126953125, + "rewards/accuracy_reward/std": 0.33324605226516724, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.83544921875, + "rewards/tag_count_reward/std": 0.28178247809410095, + "step": 177 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.205078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 1169.005859375, + "completions/mean_terminated_length": 942.23828125, + "completions/min_length": 120.0, + "completions/min_terminated_length": 120.0, + "epoch": 0.06076640778356235, + "grad_norm": 0.12574030458927155, + "kl": 0.031005859375, + "learning_rate": 6.061643835616438e-07, + "loss": 0.1577, + "num_tokens": 126289696.0, + "reward": 0.9912109375, + "reward_std": 0.3792455494403839, + "rewards/accuracy_reward/mean": 0.16796875, + "rewards/accuracy_reward/std": 0.374204158782959, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8232421875, + "rewards/tag_count_reward/std": 0.2854200601577759, + "step": 178 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.216796875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1948.0, + "completions/mean_length": 1147.837890625, + "completions/mean_terminated_length": 898.6658935546875, + "completions/min_length": 134.0, + "completions/min_terminated_length": 134.0, + "epoch": 0.06110779209695315, + "grad_norm": 0.11976183205842972, + "kl": 0.0303955078125, + "learning_rate": 6.095890410958904e-07, + "loss": 0.1866, + "num_tokens": 126951517.0, + "reward": 0.89794921875, + "reward_std": 0.30390793085098267, + "rewards/accuracy_reward/mean": 0.0947580635547638, + "rewards/accuracy_reward/std": 0.29317617416381836, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.80615234375, + "rewards/tag_count_reward/std": 0.3010845482349396, + "step": 179 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.166015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1998.0, + "completions/mean_length": 1123.68359375, + "completions/mean_terminated_length": 939.6861572265625, + "completions/min_length": 125.0, + "completions/min_terminated_length": 125.0, + "epoch": 0.06144917641034395, + "grad_norm": 0.13634192943572998, + "kl": 0.0311279296875, + "learning_rate": 6.13013698630137e-07, + "loss": 0.1713, + "num_tokens": 127615931.0, + "reward": 0.908203125, + "reward_std": 0.32293522357940674, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.837890625, + "rewards/tag_count_reward/std": 0.27642151713371277, + "step": 180 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.169921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2033.0, + "completions/mean_length": 1028.015625, + "completions/mean_terminated_length": 819.2188110351562, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.06179056072373475, + "grad_norm": 0.16368067264556885, + "kl": 0.03240966796875, + "learning_rate": 6.164383561643835e-07, + "loss": 0.2359, + "num_tokens": 128223923.0, + "reward": 0.98193359375, + "reward_std": 0.3258340358734131, + "rewards/accuracy_reward/mean": 0.140625, + "rewards/accuracy_reward/std": 0.3479743003845215, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.84130859375, + "rewards/tag_count_reward/std": 0.27150672674179077, + "step": 181 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 1265.26953125, + "completions/mean_terminated_length": 1025.658203125, + "completions/min_length": 139.0, + "completions/min_terminated_length": 139.0, + "epoch": 0.06213194503712555, + "grad_norm": 0.1233213022351265, + "kl": 0.024993896484375, + "learning_rate": 6.198630136986301e-07, + "loss": 0.1866, + "num_tokens": 128949901.0, + "reward": 0.8642578125, + "reward_std": 0.33978962898254395, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.7939453125, + "rewards/tag_count_reward/std": 0.3033125400543213, + "step": 182 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.177734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 1060.158203125, + "completions/mean_terminated_length": 846.6342163085938, + "completions/min_length": 135.0, + "completions/min_terminated_length": 135.0, + "epoch": 0.062473329350516346, + "grad_norm": 0.13886182010173798, + "kl": 0.035919189453125, + "learning_rate": 6.232876712328768e-07, + "loss": 0.1899, + "num_tokens": 129569054.0, + "reward": 0.90283203125, + "reward_std": 0.3168081045150757, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.83642578125, + "rewards/tag_count_reward/std": 0.2744435966014862, + "step": 183 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.201171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 1194.966796875, + "completions/mean_terminated_length": 980.144287109375, + "completions/min_length": 198.0, + "completions/min_terminated_length": 198.0, + "epoch": 0.06281471366390715, + "grad_norm": 0.12694279849529266, + "kl": 0.0264892578125, + "learning_rate": 6.267123287671232e-07, + "loss": 0.1723, + "num_tokens": 130256861.0, + "reward": 0.94189453125, + "reward_std": 0.3124285340309143, + "rewards/accuracy_reward/mean": 0.119140625, + "rewards/accuracy_reward/std": 0.32427072525024414, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.82275390625, + "rewards/tag_count_reward/std": 0.2900077998638153, + "step": 184 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2033.0, + "completions/mean_length": 1163.544921875, + "completions/mean_terminated_length": 948.871337890625, + "completions/min_length": 198.0, + "completions/min_terminated_length": 198.0, + "epoch": 0.06315609797729795, + "grad_norm": 0.13889499008655548, + "kl": 0.030975341796875, + "learning_rate": 6.301369863013698e-07, + "loss": 0.1728, + "num_tokens": 130932100.0, + "reward": 0.97265625, + "reward_std": 0.3136816918849945, + "rewards/accuracy_reward/mean": 0.15927419066429138, + "rewards/accuracy_reward/std": 0.3663010001182556, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.818359375, + "rewards/tag_count_reward/std": 0.2831968069076538, + "step": 185 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2019.0, + "completions/mean_length": 1212.626953125, + "completions/mean_terminated_length": 984.042236328125, + "completions/min_length": 169.0, + "completions/min_terminated_length": 169.0, + "epoch": 0.06349748229068874, + "grad_norm": 0.13234475255012512, + "kl": 0.027008056640625, + "learning_rate": 6.335616438356164e-07, + "loss": 0.1543, + "num_tokens": 131633621.0, + "reward": 1.02880859375, + "reward_std": 0.3996187746524811, + "rewards/accuracy_reward/mean": 0.212890625, + "rewards/accuracy_reward/std": 0.409751296043396, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.81591796875, + "rewards/tag_count_reward/std": 0.298685222864151, + "step": 186 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.146484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 1028.087890625, + "completions/mean_terminated_length": 853.0457153320312, + "completions/min_length": 161.0, + "completions/min_terminated_length": 161.0, + "epoch": 0.06383886660407954, + "grad_norm": 0.1387174427509308, + "kl": 0.030181884765625, + "learning_rate": 6.369863013698629e-07, + "loss": 0.1557, + "num_tokens": 132234946.0, + "reward": 1.0439453125, + "reward_std": 0.33869534730911255, + "rewards/accuracy_reward/mean": 0.185546875, + "rewards/accuracy_reward/std": 0.38912075757980347, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8583984375, + "rewards/tag_count_reward/std": 0.25977516174316406, + "step": 187 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.16015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 1202.458984375, + "completions/mean_terminated_length": 1041.21630859375, + "completions/min_length": 197.0, + "completions/min_terminated_length": 197.0, + "epoch": 0.06418025091747034, + "grad_norm": 0.1355055719614029, + "kl": 0.03045654296875, + "learning_rate": 6.404109589041096e-07, + "loss": 0.1949, + "num_tokens": 132928477.0, + "reward": 0.92333984375, + "reward_std": 0.3060658872127533, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.84326171875, + "rewards/tag_count_reward/std": 0.272641122341156, + "step": 188 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 1094.248046875, + "completions/mean_terminated_length": 896.299560546875, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "epoch": 0.06452163523086114, + "grad_norm": 0.14412641525268555, + "kl": 0.029541015625, + "learning_rate": 6.438356164383562e-07, + "loss": 0.2264, + "num_tokens": 133565260.0, + "reward": 0.9443359375, + "reward_std": 0.319268137216568, + "rewards/accuracy_reward/mean": 0.11328125, + "rewards/accuracy_reward/std": 0.3172462284564972, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8310546875, + "rewards/tag_count_reward/std": 0.28587502241134644, + "step": 189 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2005.0, + "completions/mean_length": 1186.87890625, + "completions/mean_terminated_length": 967.3775024414062, + "completions/min_length": 189.0, + "completions/min_terminated_length": 189.0, + "epoch": 0.06486301954425194, + "grad_norm": 0.12044928967952728, + "kl": 0.0277099609375, + "learning_rate": 6.472602739726027e-07, + "loss": 0.1837, + "num_tokens": 134245438.0, + "reward": 0.9306640625, + "reward_std": 0.3642945885658264, + "rewards/accuracy_reward/mean": 0.099609375, + "rewards/accuracy_reward/std": 0.29977133870124817, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8310546875, + "rewards/tag_count_reward/std": 0.2854468524456024, + "step": 190 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.166015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 1131.662109375, + "completions/mean_terminated_length": 949.2528686523438, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.06520440385764274, + "grad_norm": 0.13368235528469086, + "kl": 0.028045654296875, + "learning_rate": 6.506849315068493e-07, + "loss": 0.1476, + "num_tokens": 134906465.0, + "reward": 1.0322265625, + "reward_std": 0.31763800978660583, + "rewards/accuracy_reward/mean": 0.162109375, + "rewards/accuracy_reward/std": 0.3689115643501282, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8701171875, + "rewards/tag_count_reward/std": 0.25789982080459595, + "step": 191 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.173828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 1241.54296875, + "completions/mean_terminated_length": 1071.8629150390625, + "completions/min_length": 204.0, + "completions/min_terminated_length": 204.0, + "epoch": 0.06554578817103354, + "grad_norm": 0.11532191932201385, + "kl": 0.0250244140625, + "learning_rate": 6.541095890410958e-07, + "loss": 0.1354, + "num_tokens": 135617511.0, + "reward": 0.947265625, + "reward_std": 0.32218992710113525, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.849609375, + "rewards/tag_count_reward/std": 0.26833879947662354, + "step": 192 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.216796875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 1185.435546875, + "completions/mean_terminated_length": 946.6708374023438, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.06588717248442434, + "grad_norm": 0.13891267776489258, + "kl": 0.02862548828125, + "learning_rate": 6.575342465753423e-07, + "loss": 0.1904, + "num_tokens": 136308198.0, + "reward": 0.90380859375, + "reward_std": 0.30265724658966064, + "rewards/accuracy_reward/mean": 0.087890625, + "rewards/accuracy_reward/std": 0.2834126651287079, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.81591796875, + "rewards/tag_count_reward/std": 0.29456183314323425, + "step": 193 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.146484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 1072.626953125, + "completions/mean_terminated_length": 905.2288208007812, + "completions/min_length": 181.0, + "completions/min_terminated_length": 181.0, + "epoch": 0.06622855679781514, + "grad_norm": 0.17406651377677917, + "kl": 0.03094482421875, + "learning_rate": 6.60958904109589e-07, + "loss": 0.1819, + "num_tokens": 136935495.0, + "reward": 0.98876953125, + "reward_std": 0.3156236410140991, + "rewards/accuracy_reward/mean": 0.13306452333927155, + "rewards/accuracy_reward/std": 0.3399873673915863, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.85986328125, + "rewards/tag_count_reward/std": 0.2522217929363251, + "step": 194 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.228515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 1188.35546875, + "completions/mean_terminated_length": 933.7265625, + "completions/min_length": 154.0, + "completions/min_terminated_length": 154.0, + "epoch": 0.06656994111120594, + "grad_norm": 0.13052046298980713, + "kl": 0.03033447265625, + "learning_rate": 6.643835616438356e-07, + "loss": 0.2095, + "num_tokens": 137617373.0, + "reward": 0.916015625, + "reward_std": 0.33053910732269287, + "rewards/accuracy_reward/mean": 0.1015625, + "rewards/accuracy_reward/std": 0.30236753821372986, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.814453125, + "rewards/tag_count_reward/std": 0.2983911335468292, + "step": 195 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.17578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 1194.693359375, + "completions/mean_terminated_length": 1012.7085571289062, + "completions/min_length": 151.0, + "completions/min_terminated_length": 151.0, + "epoch": 0.06691132542459674, + "grad_norm": 0.14570492506027222, + "kl": 0.029083251953125, + "learning_rate": 6.678082191780822e-07, + "loss": 0.1595, + "num_tokens": 138305104.0, + "reward": 0.9814453125, + "reward_std": 0.30250635743141174, + "rewards/accuracy_reward/mean": 0.126953125, + "rewards/accuracy_reward/std": 0.33324605226516724, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8544921875, + "rewards/tag_count_reward/std": 0.2518412172794342, + "step": 196 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 1146.6015625, + "completions/mean_terminated_length": 969.6915283203125, + "completions/min_length": 164.0, + "completions/min_terminated_length": 164.0, + "epoch": 0.06725270973798754, + "grad_norm": 0.12312240898609161, + "kl": 0.025299072265625, + "learning_rate": 6.712328767123287e-07, + "loss": 0.166, + "num_tokens": 138967364.0, + "reward": 0.9775390625, + "reward_std": 0.31271493434906006, + "rewards/accuracy_reward/mean": 0.115234375, + "rewards/accuracy_reward/std": 0.3196168541908264, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8623046875, + "rewards/tag_count_reward/std": 0.25715774297714233, + "step": 197 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14453125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2017.0, + "completions/mean_length": 1078.677734375, + "completions/mean_terminated_length": 914.9109497070312, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "epoch": 0.06759409405137834, + "grad_norm": 0.12881354987621307, + "kl": 0.027496337890625, + "learning_rate": 6.746575342465753e-07, + "loss": 0.1802, + "num_tokens": 139590975.0, + "reward": 1.03271484375, + "reward_std": 0.3340844511985779, + "rewards/accuracy_reward/mean": 0.1484375, + "rewards/accuracy_reward/std": 0.35588082671165466, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.88427734375, + "rewards/tag_count_reward/std": 0.24463292956352234, + "step": 198 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 1096.515625, + "completions/mean_terminated_length": 925.5115356445312, + "completions/min_length": 152.0, + "completions/min_terminated_length": 152.0, + "epoch": 0.06793547836476914, + "grad_norm": 0.14294716715812683, + "kl": 0.02911376953125, + "learning_rate": 6.78082191780822e-07, + "loss": 0.1705, + "num_tokens": 140229703.0, + "reward": 1.00634765625, + "reward_std": 0.29666227102279663, + "rewards/accuracy_reward/mean": 0.140625, + "rewards/accuracy_reward/std": 0.3479743003845215, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.86572265625, + "rewards/tag_count_reward/std": 0.2539547383785248, + "step": 199 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.205078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 1200.646484375, + "completions/mean_terminated_length": 982.041748046875, + "completions/min_length": 128.0, + "completions/min_terminated_length": 128.0, + "epoch": 0.06827686267815994, + "grad_norm": 0.3884568214416504, + "kl": 0.0321044921875, + "learning_rate": 6.815068493150684e-07, + "loss": 0.1919, + "num_tokens": 140912146.0, + "reward": 0.9267578125, + "reward_std": 0.3244363069534302, + "rewards/accuracy_reward/mean": 0.099609375, + "rewards/accuracy_reward/std": 0.29977133870124817, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8271484375, + "rewards/tag_count_reward/std": 0.27830082178115845, + "step": 200 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.177734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 1189.732421875, + "completions/mean_terminated_length": 1004.2161865234375, + "completions/min_length": 249.0, + "completions/min_terminated_length": 249.0, + "epoch": 0.06861824699155074, + "grad_norm": 0.11886925250291824, + "kl": 0.026519775390625, + "learning_rate": 6.84931506849315e-07, + "loss": 0.1603, + "num_tokens": 141598809.0, + "reward": 0.94677734375, + "reward_std": 0.27545297145843506, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.85302734375, + "rewards/tag_count_reward/std": 0.26543471217155457, + "step": 201 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2024.0, + "completions/mean_length": 1105.384765625, + "completions/mean_terminated_length": 887.8582153320312, + "completions/min_length": 139.0, + "completions/min_terminated_length": 139.0, + "epoch": 0.06895963130494154, + "grad_norm": 0.1469177007675171, + "kl": 0.03057861328125, + "learning_rate": 6.883561643835616e-07, + "loss": 0.1918, + "num_tokens": 142248910.0, + "reward": 1.02099609375, + "reward_std": 0.32276904582977295, + "rewards/accuracy_reward/mean": 0.17578125, + "rewards/accuracy_reward/std": 0.3810062110424042, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.84521484375, + "rewards/tag_count_reward/std": 0.2728618085384369, + "step": 202 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.130859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2008.0, + "completions/mean_length": 1059.806640625, + "completions/mean_terminated_length": 911.0224609375, + "completions/min_length": 88.0, + "completions/min_terminated_length": 88.0, + "epoch": 0.06930101561833234, + "grad_norm": 0.16338153183460236, + "kl": 0.03179931640625, + "learning_rate": 6.917808219178081e-07, + "loss": 0.1605, + "num_tokens": 142869899.0, + "reward": 0.974609375, + "reward_std": 0.302266001701355, + "rewards/accuracy_reward/mean": 0.1015625, + "rewards/accuracy_reward/std": 0.30236753821372986, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.873046875, + "rewards/tag_count_reward/std": 0.2541169822216034, + "step": 203 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.126953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 973.978515625, + "completions/mean_terminated_length": 817.8009033203125, + "completions/min_length": 130.0, + "completions/min_terminated_length": 130.0, + "epoch": 0.06964239993172314, + "grad_norm": 0.16850519180297852, + "kl": 0.035400390625, + "learning_rate": 6.952054794520548e-07, + "loss": 0.1791, + "num_tokens": 143440848.0, + "reward": 1.0654296875, + "reward_std": 0.3052448332309723, + "rewards/accuracy_reward/mean": 0.185546875, + "rewards/accuracy_reward/std": 0.38912075757980347, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8798828125, + "rewards/tag_count_reward/std": 0.25214457511901855, + "step": 204 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.193359375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 1150.9921875, + "completions/mean_terminated_length": 935.9710083007812, + "completions/min_length": 179.0, + "completions/min_terminated_length": 179.0, + "epoch": 0.06998378424511394, + "grad_norm": 0.11850355565547943, + "kl": 0.026214599609375, + "learning_rate": 6.986301369863014e-07, + "loss": 0.1802, + "num_tokens": 144103548.0, + "reward": 0.9375, + "reward_std": 0.3141724467277527, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.845703125, + "rewards/tag_count_reward/std": 0.2778007984161377, + "step": 205 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.16015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 1131.931640625, + "completions/mean_terminated_length": 957.239501953125, + "completions/min_length": 261.0, + "completions/min_terminated_length": 261.0, + "epoch": 0.07032516855850474, + "grad_norm": 0.13232004642486572, + "kl": 0.029022216796875, + "learning_rate": 7.02054794520548e-07, + "loss": 0.1503, + "num_tokens": 144768345.0, + "reward": 0.9990234375, + "reward_std": 0.3439529538154602, + "rewards/accuracy_reward/mean": 0.126953125, + "rewards/accuracy_reward/std": 0.33324605226516724, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8720703125, + "rewards/tag_count_reward/std": 0.2531430721282959, + "step": 206 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2017.0, + "completions/mean_length": 1070.21484375, + "completions/mean_terminated_length": 899.7752075195312, + "completions/min_length": 163.0, + "completions/min_terminated_length": 163.0, + "epoch": 0.07066655287189554, + "grad_norm": 0.13105420768260956, + "kl": 0.030609130859375, + "learning_rate": 7.054794520547945e-07, + "loss": 0.1305, + "num_tokens": 145391687.0, + "reward": 0.9775390625, + "reward_std": 0.2806186378002167, + "rewards/accuracy_reward/mean": 0.103515625, + "rewards/accuracy_reward/std": 0.30492907762527466, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8740234375, + "rewards/tag_count_reward/std": 0.255562424659729, + "step": 207 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.146484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2033.0, + "completions/mean_length": 1061.9375, + "completions/mean_terminated_length": 892.7047729492188, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.07100793718528634, + "grad_norm": 0.28645065426826477, + "kl": 0.03521728515625, + "learning_rate": 7.08904109589041e-07, + "loss": 0.1834, + "num_tokens": 146008183.0, + "reward": 1.04541015625, + "reward_std": 0.308370977640152, + "rewards/accuracy_reward/mean": 0.17578125, + "rewards/accuracy_reward/std": 0.3810062110424042, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.86962890625, + "rewards/tag_count_reward/std": 0.25213465094566345, + "step": 208 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.166015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 1117.779296875, + "completions/mean_terminated_length": 932.6065063476562, + "completions/min_length": 210.0, + "completions/min_terminated_length": 210.0, + "epoch": 0.07134932149867713, + "grad_norm": 0.14213435351848602, + "kl": 0.0330810546875, + "learning_rate": 7.123287671232876e-07, + "loss": 0.182, + "num_tokens": 146655126.0, + "reward": 1.01025390625, + "reward_std": 0.3315596878528595, + "rewards/accuracy_reward/mean": 0.162109375, + "rewards/accuracy_reward/std": 0.3689115643501282, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.84814453125, + "rewards/tag_count_reward/std": 0.2758397161960602, + "step": 209 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.146484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 1069.072265625, + "completions/mean_terminated_length": 901.0640258789062, + "completions/min_length": 209.0, + "completions/min_terminated_length": 209.0, + "epoch": 0.07169070581206793, + "grad_norm": 0.15966561436653137, + "kl": 0.03521728515625, + "learning_rate": 7.157534246575342e-07, + "loss": 0.2082, + "num_tokens": 147271755.0, + "reward": 0.92529296875, + "reward_std": 0.29846563935279846, + "rewards/accuracy_reward/mean": 0.06451612710952759, + "rewards/accuracy_reward/std": 0.2459181249141693, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.86279296875, + "rewards/tag_count_reward/std": 0.2623606026172638, + "step": 210 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14453125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 1088.546875, + "completions/mean_terminated_length": 926.4474487304688, + "completions/min_length": 203.0, + "completions/min_terminated_length": 203.0, + "epoch": 0.07203209012545873, + "grad_norm": 0.1488552987575531, + "kl": 0.0316162109375, + "learning_rate": 7.191780821917808e-07, + "loss": 0.1847, + "num_tokens": 147906131.0, + "reward": 1.0498046875, + "reward_std": 0.3397788405418396, + "rewards/accuracy_reward/mean": 0.189453125, + "rewards/accuracy_reward/std": 0.3922513723373413, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8603515625, + "rewards/tag_count_reward/std": 0.26176854968070984, + "step": 211 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 1117.904296875, + "completions/mean_terminated_length": 950.7442626953125, + "completions/min_length": 208.0, + "completions/min_terminated_length": 208.0, + "epoch": 0.07237347443884953, + "grad_norm": 0.8496732711791992, + "kl": 0.05059814453125, + "learning_rate": 7.226027397260274e-07, + "loss": 0.1585, + "num_tokens": 148560514.0, + "reward": 0.95263671875, + "reward_std": 0.3026370406150818, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.85888671875, + "rewards/tag_count_reward/std": 0.26308074593544006, + "step": 212 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.166015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2029.0, + "completions/mean_length": 1193.779296875, + "completions/mean_terminated_length": 1023.7352905273438, + "completions/min_length": 190.0, + "completions/min_terminated_length": 190.0, + "epoch": 0.07271485875224033, + "grad_norm": 0.1313561499118805, + "kl": 0.026763916015625, + "learning_rate": 7.260273972602739e-07, + "loss": 0.18, + "num_tokens": 149258769.0, + "reward": 0.9873046875, + "reward_std": 0.33825457096099854, + "rewards/accuracy_reward/mean": 0.134765625, + "rewards/accuracy_reward/std": 0.3418070077896118, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8525390625, + "rewards/tag_count_reward/std": 0.26905539631843567, + "step": 213 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.099609375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1982.0, + "completions/mean_length": 1029.296875, + "completions/mean_terminated_length": 916.5986938476562, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "epoch": 0.07305624306563113, + "grad_norm": 0.1539052575826645, + "kl": 0.032440185546875, + "learning_rate": 7.294520547945205e-07, + "loss": 0.1719, + "num_tokens": 149857833.0, + "reward": 0.92626953125, + "reward_std": 0.24862925708293915, + "rewards/accuracy_reward/mean": 0.032258063554763794, + "rewards/accuracy_reward/std": 0.17686307430267334, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.89501953125, + "rewards/tag_count_reward/std": 0.2284543514251709, + "step": 214 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.142578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 1090.591796875, + "completions/mean_terminated_length": 931.3872680664062, + "completions/min_length": 162.0, + "completions/min_terminated_length": 162.0, + "epoch": 0.07339762737902193, + "grad_norm": 0.13333448767662048, + "kl": 0.03228759765625, + "learning_rate": 7.328767123287672e-07, + "loss": 0.1463, + "num_tokens": 150495608.0, + "reward": 0.96533203125, + "reward_std": 0.2588098347187042, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.86962890625, + "rewards/tag_count_reward/std": 0.25978022813796997, + "step": 215 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.142578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2033.0, + "completions/mean_length": 1102.90234375, + "completions/mean_terminated_length": 945.7449340820312, + "completions/min_length": 143.0, + "completions/min_terminated_length": 143.0, + "epoch": 0.07373901169241273, + "grad_norm": 0.15049462020397186, + "kl": 0.03265380859375, + "learning_rate": 7.363013698630136e-07, + "loss": 0.1798, + "num_tokens": 151135670.0, + "reward": 1.0068359375, + "reward_std": 0.30560004711151123, + "rewards/accuracy_reward/mean": 0.130859375, + "rewards/accuracy_reward/std": 0.33757632970809937, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8759765625, + "rewards/tag_count_reward/std": 0.2579442858695984, + "step": 216 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2007.0, + "completions/mean_length": 1070.8828125, + "completions/mean_terminated_length": 916.1358032226562, + "completions/min_length": 182.0, + "completions/min_terminated_length": 182.0, + "epoch": 0.07408039600580353, + "grad_norm": 0.13311637938022614, + "kl": 0.028564453125, + "learning_rate": 7.397260273972602e-07, + "loss": 0.1435, + "num_tokens": 151760234.0, + "reward": 1.0458984375, + "reward_std": 0.30893051624298096, + "rewards/accuracy_reward/mean": 0.16796875, + "rewards/accuracy_reward/std": 0.374204158782959, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8779296875, + "rewards/tag_count_reward/std": 0.2536257803440094, + "step": 217 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.146484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 1083.294921875, + "completions/mean_terminated_length": 917.7276611328125, + "completions/min_length": 161.0, + "completions/min_terminated_length": 161.0, + "epoch": 0.07442178031919433, + "grad_norm": 0.15257856249809265, + "kl": 0.03009033203125, + "learning_rate": 7.431506849315068e-07, + "loss": 0.2131, + "num_tokens": 152389297.0, + "reward": 1.00390625, + "reward_std": 0.2963552474975586, + "rewards/accuracy_reward/mean": 0.13671875, + "rewards/accuracy_reward/std": 0.3438861668109894, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8671875, + "rewards/tag_count_reward/std": 0.25544461607933044, + "step": 218 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.140625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2019.0, + "completions/mean_length": 1074.134765625, + "completions/mean_terminated_length": 914.7749633789062, + "completions/min_length": 127.0, + "completions/min_terminated_length": 127.0, + "epoch": 0.07476316463258513, + "grad_norm": 0.1511964648962021, + "kl": 0.030914306640625, + "learning_rate": 7.465753424657533e-07, + "loss": 0.2, + "num_tokens": 153029862.0, + "reward": 0.98779296875, + "reward_std": 0.3228898048400879, + "rewards/accuracy_reward/mean": 0.11328125, + "rewards/accuracy_reward/std": 0.3172462284564972, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.87451171875, + "rewards/tag_count_reward/std": 0.25315961241722107, + "step": 219 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.107421875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 1098.11328125, + "completions/mean_terminated_length": 983.7943115234375, + "completions/min_length": 236.0, + "completions/min_terminated_length": 236.0, + "epoch": 0.07510454894597593, + "grad_norm": 0.13734576106071472, + "kl": 0.027099609375, + "learning_rate": 7.5e-07, + "loss": 0.172, + "num_tokens": 153673248.0, + "reward": 0.97900390625, + "reward_std": 0.26450347900390625, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.91259765625, + "rewards/tag_count_reward/std": 0.21513772010803223, + "step": 220 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.138671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 1134.908203125, + "completions/mean_terminated_length": 987.9025268554688, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.07544593325936673, + "grad_norm": 0.13219542801380157, + "kl": 0.027252197265625, + "learning_rate": 7.534246575342466e-07, + "loss": 0.1591, + "num_tokens": 154327041.0, + "reward": 1.0224609375, + "reward_std": 0.29937922954559326, + "rewards/accuracy_reward/mean": 0.15234375, + "rewards/accuracy_reward/std": 0.35970520973205566, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8701171875, + "rewards/tag_count_reward/std": 0.24774044752120972, + "step": 221 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14453125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2028.0, + "completions/mean_length": 1079.619140625, + "completions/mean_terminated_length": 916.0113525390625, + "completions/min_length": 174.0, + "completions/min_terminated_length": 174.0, + "epoch": 0.07578731757275753, + "grad_norm": 0.1405760943889618, + "kl": 0.03057861328125, + "learning_rate": 7.568493150684932e-07, + "loss": 0.2047, + "num_tokens": 154958494.0, + "reward": 1.04248046875, + "reward_std": 0.28535062074661255, + "rewards/accuracy_reward/mean": 0.158203125, + "rewards/accuracy_reward/std": 0.36528825759887695, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.88427734375, + "rewards/tag_count_reward/std": 0.2385578453540802, + "step": 222 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2019.0, + "completions/mean_length": 1130.537109375, + "completions/mean_terminated_length": 970.6123657226562, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "epoch": 0.07612870188614833, + "grad_norm": 0.13484491407871246, + "kl": 0.031005859375, + "learning_rate": 7.602739726027397e-07, + "loss": 0.1306, + "num_tokens": 155615889.0, + "reward": 1.099609375, + "reward_std": 0.3330080509185791, + "rewards/accuracy_reward/mean": 0.21484375, + "rewards/accuracy_reward/std": 0.4111155867576599, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.884765625, + "rewards/tag_count_reward/std": 0.2446138858795166, + "step": 223 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.146484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2016.0, + "completions/mean_length": 1042.552734375, + "completions/mean_terminated_length": 869.9931030273438, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.07647008619953913, + "grad_norm": 0.1428431272506714, + "kl": 0.029296875, + "learning_rate": 7.636986301369863e-07, + "loss": 0.1659, + "num_tokens": 156226780.0, + "reward": 0.94140625, + "reward_std": 0.2550290822982788, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.875, + "rewards/tag_count_reward/std": 0.2565196752548218, + "step": 224 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 1103.482421875, + "completions/mean_terminated_length": 938.8416748046875, + "completions/min_length": 197.0, + "completions/min_terminated_length": 197.0, + "epoch": 0.07681147051292993, + "grad_norm": 0.14843755960464478, + "kl": 0.029388427734375, + "learning_rate": 7.671232876712328e-07, + "loss": 0.1851, + "num_tokens": 156872675.0, + "reward": 0.99462890625, + "reward_std": 0.3229602575302124, + "rewards/accuracy_reward/mean": 0.12890625, + "rewards/accuracy_reward/std": 0.33542385697364807, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.86572265625, + "rewards/tag_count_reward/std": 0.25730398297309875, + "step": 225 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08984375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1992.0, + "completions/mean_length": 960.49609375, + "completions/mean_terminated_length": 853.1459350585938, + "completions/min_length": 102.0, + "completions/min_terminated_length": 102.0, + "epoch": 0.07715285482632073, + "grad_norm": 0.16743634641170502, + "kl": 0.033355712890625, + "learning_rate": 7.705479452054794e-07, + "loss": 0.1358, + "num_tokens": 157447953.0, + "reward": 1.10009765625, + "reward_std": 0.2929975390434265, + "rewards/accuracy_reward/mean": 0.1895161271095276, + "rewards/accuracy_reward/std": 0.39231374859809875, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.91650390625, + "rewards/tag_count_reward/std": 0.20922017097473145, + "step": 226 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.201171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1991.0, + "completions/mean_length": 1157.2578125, + "completions/mean_terminated_length": 932.9389038085938, + "completions/min_length": 200.0, + "completions/min_terminated_length": 200.0, + "epoch": 0.07749423913971153, + "grad_norm": 0.14617608487606049, + "kl": 0.0316162109375, + "learning_rate": 7.73972602739726e-07, + "loss": 0.1841, + "num_tokens": 158119637.0, + "reward": 0.91748046875, + "reward_std": 0.332801878452301, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.83154296875, + "rewards/tag_count_reward/std": 0.29562392830848694, + "step": 227 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2009.0, + "completions/mean_length": 1132.5859375, + "completions/mean_terminated_length": 952.9252319335938, + "completions/min_length": 110.0, + "completions/min_terminated_length": 110.0, + "epoch": 0.07783562345310233, + "grad_norm": 1.4102917909622192, + "kl": 0.04827880859375, + "learning_rate": 7.773972602739726e-07, + "loss": 0.1814, + "num_tokens": 158770721.0, + "reward": 1.0283203125, + "reward_std": 0.34487906098365784, + "rewards/accuracy_reward/mean": 0.158203125, + "rewards/accuracy_reward/std": 0.36528825759887695, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8701171875, + "rewards/tag_count_reward/std": 0.25262919068336487, + "step": 228 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.189453125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2016.0, + "completions/mean_length": 1147.451171875, + "completions/mean_terminated_length": 936.9614868164062, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.07817700776649313, + "grad_norm": 0.1398928165435791, + "kl": 0.030181884765625, + "learning_rate": 7.808219178082191e-07, + "loss": 0.1918, + "num_tokens": 159442840.0, + "reward": 1.0068359375, + "reward_std": 0.36788713932037354, + "rewards/accuracy_reward/mean": 0.16935484111309052, + "rewards/accuracy_reward/std": 0.3754436671733856, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8427734375, + "rewards/tag_count_reward/std": 0.27659255266189575, + "step": 229 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2019.0, + "completions/mean_length": 1174.890625, + "completions/mean_terminated_length": 941.485107421875, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "epoch": 0.07851839207988393, + "grad_norm": 0.15699157118797302, + "kl": 0.0335693359375, + "learning_rate": 7.842465753424657e-07, + "loss": 0.2296, + "num_tokens": 160128288.0, + "reward": 0.8662109375, + "reward_std": 0.31422632932662964, + "rewards/accuracy_reward/mean": 0.04435483738780022, + "rewards/accuracy_reward/std": 0.2060900777578354, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8232421875, + "rewards/tag_count_reward/std": 0.2959393262863159, + "step": 230 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.22265625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 1187.904296875, + "completions/mean_terminated_length": 941.5452270507812, + "completions/min_length": 182.0, + "completions/min_terminated_length": 182.0, + "epoch": 0.07885977639327472, + "grad_norm": 0.14857465028762817, + "kl": 0.03204345703125, + "learning_rate": 7.876712328767124e-07, + "loss": 0.18, + "num_tokens": 160812847.0, + "reward": 0.95654296875, + "reward_std": 0.3289763033390045, + "rewards/accuracy_reward/mean": 0.1328125, + "rewards/accuracy_reward/std": 0.33970388770103455, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.82373046875, + "rewards/tag_count_reward/std": 0.2914441227912903, + "step": 231 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.119140625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 1065.884765625, + "completions/mean_terminated_length": 933.0487670898438, + "completions/min_length": 148.0, + "completions/min_terminated_length": 148.0, + "epoch": 0.07920116070666552, + "grad_norm": 0.13972456753253937, + "kl": 0.03094482421875, + "learning_rate": 7.91095890410959e-07, + "loss": 0.1981, + "num_tokens": 161438404.0, + "reward": 0.96533203125, + "reward_std": 0.24260011315345764, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.89501953125, + "rewards/tag_count_reward/std": 0.23217174410820007, + "step": 232 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2025.0, + "completions/mean_length": 1175.013671875, + "completions/mean_terminated_length": 973.5552978515625, + "completions/min_length": 240.0, + "completions/min_terminated_length": 240.0, + "epoch": 0.07954254502005632, + "grad_norm": 0.7790918946266174, + "kl": 0.036651611328125, + "learning_rate": 7.945205479452054e-07, + "loss": 0.1494, + "num_tokens": 162115835.0, + "reward": 1.0009765625, + "reward_std": 0.34966665506362915, + "rewards/accuracy_reward/mean": 0.16015625, + "rewards/accuracy_reward/std": 0.3671095669269562, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8408203125, + "rewards/tag_count_reward/std": 0.2863559424877167, + "step": 233 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12890625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 1065.4765625, + "completions/mean_terminated_length": 920.0807495117188, + "completions/min_length": 31.0, + "completions/min_terminated_length": 31.0, + "epoch": 0.07988392933344712, + "grad_norm": 0.18480327725410461, + "kl": 0.0311279296875, + "learning_rate": 7.97945205479452e-07, + "loss": 0.2279, + "num_tokens": 162742239.0, + "reward": 0.95263671875, + "reward_std": 0.2685253620147705, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.88818359375, + "rewards/tag_count_reward/std": 0.23107852041721344, + "step": 234 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.189453125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2031.0, + "completions/mean_length": 1140.896484375, + "completions/mean_terminated_length": 928.8746948242188, + "completions/min_length": 195.0, + "completions/min_terminated_length": 195.0, + "epoch": 0.08022531364683792, + "grad_norm": 0.11098338663578033, + "kl": 0.02996826171875, + "learning_rate": 8.013698630136985e-07, + "loss": 0.1437, + "num_tokens": 163399050.0, + "reward": 0.99462890625, + "reward_std": 0.3432961106300354, + "rewards/accuracy_reward/mean": 0.1484375, + "rewards/accuracy_reward/std": 0.35588082671165466, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.84619140625, + "rewards/tag_count_reward/std": 0.2813514769077301, + "step": 235 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.115234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2030.0, + "completions/mean_length": 1028.30078125, + "completions/mean_terminated_length": 895.4922485351562, + "completions/min_length": 205.0, + "completions/min_terminated_length": 205.0, + "epoch": 0.08056669796022872, + "grad_norm": 0.14085927605628967, + "kl": 0.03192138671875, + "learning_rate": 8.047945205479451e-07, + "loss": 0.1663, + "num_tokens": 163999924.0, + "reward": 1.03271484375, + "reward_std": 0.3006260395050049, + "rewards/accuracy_reward/mean": 0.150390625, + "rewards/accuracy_reward/std": 0.35780346393585205, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.88232421875, + "rewards/tag_count_reward/std": 0.2421872466802597, + "step": 236 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 1206.373046875, + "completions/mean_terminated_length": 1055.1129150390625, + "completions/min_length": 171.0, + "completions/min_terminated_length": 171.0, + "epoch": 0.08090808227361952, + "grad_norm": 0.11820080131292343, + "kl": 0.029144287109375, + "learning_rate": 8.082191780821918e-07, + "loss": 0.1273, + "num_tokens": 164700675.0, + "reward": 1.02783203125, + "reward_std": 0.3371480107307434, + "rewards/accuracy_reward/mean": 0.162109375, + "rewards/accuracy_reward/std": 0.3689115643501282, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.86572265625, + "rewards/tag_count_reward/std": 0.2490920126438141, + "step": 237 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.189453125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2033.0, + "completions/mean_length": 1126.564453125, + "completions/mean_terminated_length": 911.1928100585938, + "completions/min_length": 109.0, + "completions/min_terminated_length": 109.0, + "epoch": 0.08124946658701032, + "grad_norm": 3.9634222984313965, + "kl": 0.07635498046875, + "learning_rate": 8.116438356164384e-07, + "loss": 0.1334, + "num_tokens": 165351540.0, + "reward": 0.95947265625, + "reward_std": 0.2637588679790497, + "rewards/accuracy_reward/mean": 0.107421875, + "rewards/accuracy_reward/std": 0.30995169281959534, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.85205078125, + "rewards/tag_count_reward/std": 0.2766353189945221, + "step": 238 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 1002.380859375, + "completions/mean_terminated_length": 908.9425048828125, + "completions/min_length": 233.0, + "completions/min_terminated_length": 233.0, + "epoch": 0.08159085090040112, + "grad_norm": 0.15155275166034698, + "kl": 0.032318115234375, + "learning_rate": 8.150684931506849e-07, + "loss": 0.0906, + "num_tokens": 165945431.0, + "reward": 1.052734375, + "reward_std": 0.2858385145664215, + "rewards/accuracy_reward/mean": 0.14453125, + "rewards/accuracy_reward/std": 0.35197147727012634, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.908203125, + "rewards/tag_count_reward/std": 0.21358256042003632, + "step": 239 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2031.0, + "completions/mean_length": 1200.95703125, + "completions/mean_terminated_length": 985.0441284179688, + "completions/min_length": 198.0, + "completions/min_terminated_length": 198.0, + "epoch": 0.08193223521379192, + "grad_norm": 0.1461395025253296, + "kl": 0.03375244140625, + "learning_rate": 8.184931506849315e-07, + "loss": 0.2138, + "num_tokens": 166633793.0, + "reward": 0.9755859375, + "reward_std": 0.33675751090049744, + "rewards/accuracy_reward/mean": 0.14516128599643707, + "rewards/accuracy_reward/std": 0.3526190221309662, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8349609375, + "rewards/tag_count_reward/std": 0.2885763645172119, + "step": 240 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.134765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1947.0, + "completions/mean_length": 1022.099609375, + "completions/mean_terminated_length": 862.3092651367188, + "completions/min_length": 133.0, + "completions/min_terminated_length": 133.0, + "epoch": 0.08227361952718272, + "grad_norm": 0.152372807264328, + "kl": 0.03533935546875, + "learning_rate": 8.21917808219178e-07, + "loss": 0.1733, + "num_tokens": 167230468.0, + "reward": 1.01318359375, + "reward_std": 0.3202892541885376, + "rewards/accuracy_reward/mean": 0.1328125, + "rewards/accuracy_reward/std": 0.33970388770103455, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.88037109375, + "rewards/tag_count_reward/std": 0.2477303296327591, + "step": 241 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2033.0, + "completions/mean_length": 1018.638671875, + "completions/mean_terminated_length": 876.8155517578125, + "completions/min_length": 119.0, + "completions/min_terminated_length": 119.0, + "epoch": 0.08261500384057352, + "grad_norm": 0.14142490923404694, + "kl": 0.031280517578125, + "learning_rate": 8.253424657534246e-07, + "loss": 0.127, + "num_tokens": 167830507.0, + "reward": 1.02880859375, + "reward_std": 0.27235502004623413, + "rewards/accuracy_reward/mean": 0.1328125, + "rewards/accuracy_reward/std": 0.33970388770103455, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.89599609375, + "rewards/tag_count_reward/std": 0.23049886524677277, + "step": 242 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2029.0, + "completions/mean_length": 1119.408203125, + "completions/mean_terminated_length": 957.5435180664062, + "completions/min_length": 165.0, + "completions/min_terminated_length": 165.0, + "epoch": 0.08295638815396432, + "grad_norm": 0.1320219784975052, + "kl": 0.031768798828125, + "learning_rate": 8.287671232876712e-07, + "loss": 0.137, + "num_tokens": 168483116.0, + "reward": 0.9833984375, + "reward_std": 0.2867240011692047, + "rewards/accuracy_reward/mean": 0.123046875, + "rewards/accuracy_reward/std": 0.32881227135658264, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8603515625, + "rewards/tag_count_reward/std": 0.26868578791618347, + "step": 243 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.130859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 1146.990234375, + "completions/mean_terminated_length": 1011.3325805664062, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "epoch": 0.08329777246735512, + "grad_norm": 0.12730756402015686, + "kl": 0.0306396484375, + "learning_rate": 8.321917808219178e-07, + "loss": 0.1091, + "num_tokens": 169149015.0, + "reward": 0.97607421875, + "reward_std": 0.3049416244029999, + "rewards/accuracy_reward/mean": 0.1015625, + "rewards/accuracy_reward/std": 0.30236753821372986, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.87451171875, + "rewards/tag_count_reward/std": 0.25073230266571045, + "step": 244 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.134765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 1037.759765625, + "completions/mean_terminated_length": 880.4085693359375, + "completions/min_length": 99.0, + "completions/min_terminated_length": 99.0, + "epoch": 0.08363915678074592, + "grad_norm": 0.15358765423297882, + "kl": 0.0301513671875, + "learning_rate": 8.356164383561643e-07, + "loss": 0.1985, + "num_tokens": 169754604.0, + "reward": 0.96484375, + "reward_std": 0.29754340648651123, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8828125, + "rewards/tag_count_reward/std": 0.2461792230606079, + "step": 245 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1990.0, + "completions/mean_length": 1161.271484375, + "completions/mean_terminated_length": 987.2406005859375, + "completions/min_length": 183.0, + "completions/min_terminated_length": 183.0, + "epoch": 0.08398054109413672, + "grad_norm": 0.2361176609992981, + "kl": 0.03265380859375, + "learning_rate": 8.390410958904109e-07, + "loss": 0.1784, + "num_tokens": 170428327.0, + "reward": 0.94482421875, + "reward_std": 0.28001147508621216, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.85888671875, + "rewards/tag_count_reward/std": 0.26493385434150696, + "step": 246 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1999.0, + "completions/mean_length": 1140.546875, + "completions/mean_terminated_length": 1015.52001953125, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.08432192540752752, + "grad_norm": 0.14363712072372437, + "kl": 0.0302734375, + "learning_rate": 8.424657534246576e-07, + "loss": 0.1529, + "num_tokens": 171094079.0, + "reward": 1.06689453125, + "reward_std": 0.3600861132144928, + "rewards/accuracy_reward/mean": 0.171875, + "rewards/accuracy_reward/std": 0.3776407241821289, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.89501953125, + "rewards/tag_count_reward/std": 0.23269794881343842, + "step": 247 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12890625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2019.0, + "completions/mean_length": 1115.400390625, + "completions/mean_terminated_length": 977.3924560546875, + "completions/min_length": 158.0, + "completions/min_terminated_length": 158.0, + "epoch": 0.08466330972091832, + "grad_norm": 0.15197043120861053, + "kl": 0.0330810546875, + "learning_rate": 8.458904109589042e-07, + "loss": 0.1532, + "num_tokens": 171746684.0, + "reward": 1.00927734375, + "reward_std": 0.28392645716667175, + "rewards/accuracy_reward/mean": 0.123046875, + "rewards/accuracy_reward/std": 0.32881227135658264, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.88623046875, + "rewards/tag_count_reward/std": 0.23693007230758667, + "step": 248 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.134765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2029.0, + "completions/mean_length": 1025.802734375, + "completions/mean_terminated_length": 866.5891723632812, + "completions/min_length": 162.0, + "completions/min_terminated_length": 162.0, + "epoch": 0.08500469403430912, + "grad_norm": 0.15088102221488953, + "kl": 0.033111572265625, + "learning_rate": 8.493150684931506e-07, + "loss": 0.1955, + "num_tokens": 172352775.0, + "reward": 0.99365234375, + "reward_std": 0.2636690139770508, + "rewards/accuracy_reward/mean": 0.11088709533214569, + "rewards/accuracy_reward/std": 0.3143092691898346, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.88623046875, + "rewards/tag_count_reward/std": 0.25048065185546875, + "step": 249 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1985.0, + "completions/mean_length": 1015.931640625, + "completions/mean_terminated_length": 909.1659545898438, + "completions/min_length": 198.0, + "completions/min_terminated_length": 198.0, + "epoch": 0.08534607834769992, + "grad_norm": 0.13097302615642548, + "kl": 0.03509521484375, + "learning_rate": 8.527397260273972e-07, + "loss": 0.12, + "num_tokens": 172945076.0, + "reward": 1.02294921875, + "reward_std": 0.25544142723083496, + "rewards/accuracy_reward/mean": 0.11666666716337204, + "rewards/accuracy_reward/std": 0.3213576078414917, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.91357421875, + "rewards/tag_count_reward/std": 0.2109440118074417, + "step": 250 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2020.0, + "completions/mean_length": 1058.796875, + "completions/mean_terminated_length": 942.1659545898438, + "completions/min_length": 146.0, + "completions/min_terminated_length": 146.0, + "epoch": 0.08568746266109073, + "grad_norm": 0.14348679780960083, + "kl": 0.032196044921875, + "learning_rate": 8.561643835616438e-07, + "loss": 0.1364, + "num_tokens": 173562732.0, + "reward": 1.021484375, + "reward_std": 0.2810960114002228, + "rewards/accuracy_reward/mean": 0.125, + "rewards/accuracy_reward/std": 0.3310423493385315, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.896484375, + "rewards/tag_count_reward/std": 0.22885605692863464, + "step": 251 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.138671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 1133.736328125, + "completions/mean_terminated_length": 986.5419921875, + "completions/min_length": 201.0, + "completions/min_terminated_length": 201.0, + "epoch": 0.08602884697448153, + "grad_norm": 16.952478408813477, + "kl": 0.17535400390625, + "learning_rate": 8.595890410958903e-07, + "loss": 0.1573, + "num_tokens": 174229877.0, + "reward": 0.9892578125, + "reward_std": 0.27832746505737305, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8935546875, + "rewards/tag_count_reward/std": 0.22096005082130432, + "step": 252 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.146484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 1072.92578125, + "completions/mean_terminated_length": 905.5789184570312, + "completions/min_length": 124.0, + "completions/min_terminated_length": 124.0, + "epoch": 0.08637023128787233, + "grad_norm": 0.1386740356683731, + "kl": 0.03204345703125, + "learning_rate": 8.63013698630137e-07, + "loss": 0.2156, + "num_tokens": 174868047.0, + "reward": 0.95361328125, + "reward_std": 0.2634553909301758, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.87939453125, + "rewards/tag_count_reward/std": 0.24225826561450958, + "step": 253 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1328125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2011.0, + "completions/mean_length": 1068.017578125, + "completions/mean_terminated_length": 917.93017578125, + "completions/min_length": 123.0, + "completions/min_terminated_length": 123.0, + "epoch": 0.08671161560126313, + "grad_norm": 6.681758880615234, + "kl": 0.114501953125, + "learning_rate": 8.664383561643836e-07, + "loss": 0.153, + "num_tokens": 175483912.0, + "reward": 1.0986328125, + "reward_std": 0.34474465250968933, + "rewards/accuracy_reward/mean": 0.21484375, + "rewards/accuracy_reward/std": 0.4111155867576599, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8837890625, + "rewards/tag_count_reward/std": 0.24910975992679596, + "step": 254 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12890625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2016.0, + "completions/mean_length": 1046.703125, + "completions/mean_terminated_length": 898.5291748046875, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "epoch": 0.08705299991465393, + "grad_norm": 0.1366773396730423, + "kl": 0.032012939453125, + "learning_rate": 8.698630136986301e-07, + "loss": 0.1301, + "num_tokens": 176103568.0, + "reward": 0.99609375, + "reward_std": 0.3028711676597595, + "rewards/accuracy_reward/mean": 0.11693548411130905, + "rewards/accuracy_reward/std": 0.3216678202152252, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8828125, + "rewards/tag_count_reward/std": 0.24115973711013794, + "step": 255 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08984375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 1050.291015625, + "completions/mean_terminated_length": 951.8047485351562, + "completions/min_length": 169.0, + "completions/min_terminated_length": 169.0, + "epoch": 0.08739438422804473, + "grad_norm": 0.1439121514558792, + "kl": 0.034271240234375, + "learning_rate": 8.732876712328767e-07, + "loss": 0.1452, + "num_tokens": 176720821.0, + "reward": 1.0263671875, + "reward_std": 0.2726992666721344, + "rewards/accuracy_reward/mean": 0.123046875, + "rewards/accuracy_reward/std": 0.32881227135658264, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9033203125, + "rewards/tag_count_reward/std": 0.21824489533901215, + "step": 256 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12890625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 1063.634765625, + "completions/mean_terminated_length": 917.9664306640625, + "completions/min_length": 140.0, + "completions/min_terminated_length": 140.0, + "epoch": 0.08773576854143553, + "grad_norm": 0.15800635516643524, + "kl": 0.033660888671875, + "learning_rate": 8.767123287671232e-07, + "loss": 0.1738, + "num_tokens": 177340602.0, + "reward": 1.02734375, + "reward_std": 0.3319050967693329, + "rewards/accuracy_reward/mean": 0.140625, + "rewards/accuracy_reward/std": 0.3479743003845215, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.88671875, + "rewards/tag_count_reward/std": 0.24849718809127808, + "step": 257 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 1123.80078125, + "completions/mean_terminated_length": 977.4344482421875, + "completions/min_length": 184.0, + "completions/min_terminated_length": 184.0, + "epoch": 0.08807715285482633, + "grad_norm": 0.12673071026802063, + "kl": 0.030181884765625, + "learning_rate": 8.801369863013698e-07, + "loss": 0.1611, + "num_tokens": 177984964.0, + "reward": 0.96875, + "reward_std": 0.24542951583862305, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.890625, + "rewards/tag_count_reward/std": 0.2392502874135971, + "step": 258 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.134765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2003.0, + "completions/mean_length": 1113.703125, + "completions/mean_terminated_length": 968.1806030273438, + "completions/min_length": 102.0, + "completions/min_terminated_length": 102.0, + "epoch": 0.08841853716821713, + "grad_norm": 132.47552490234375, + "kl": 1.1121826171875, + "learning_rate": 8.835616438356164e-07, + "loss": 0.2124, + "num_tokens": 178629612.0, + "reward": 1.0703125, + "reward_std": 0.3268173933029175, + "rewards/accuracy_reward/mean": 0.1796875, + "rewards/accuracy_reward/std": 0.38430243730545044, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.890625, + "rewards/tag_count_reward/std": 0.2428024560213089, + "step": 259 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2029.0, + "completions/mean_length": 1119.72265625, + "completions/mean_terminated_length": 905.5048217773438, + "completions/min_length": 187.0, + "completions/min_terminated_length": 187.0, + "epoch": 0.08875992148160793, + "grad_norm": 0.15137813985347748, + "kl": 0.03277587890625, + "learning_rate": 8.86986301369863e-07, + "loss": 0.2252, + "num_tokens": 179282174.0, + "reward": 0.900390625, + "reward_std": 0.2901911735534668, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.853515625, + "rewards/tag_count_reward/std": 0.27631086111068726, + "step": 260 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.134765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 1087.89453125, + "completions/mean_terminated_length": 938.3521728515625, + "completions/min_length": 102.0, + "completions/min_terminated_length": 102.0, + "epoch": 0.08910130579499873, + "grad_norm": 0.15067681670188904, + "kl": 0.0322265625, + "learning_rate": 8.904109589041095e-07, + "loss": 0.1344, + "num_tokens": 179913368.0, + "reward": 1.00244140625, + "reward_std": 0.2927597761154175, + "rewards/accuracy_reward/mean": 0.11328125, + "rewards/accuracy_reward/std": 0.3172462284564972, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.88916015625, + "rewards/tag_count_reward/std": 0.2403615117073059, + "step": 261 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2012.0, + "completions/mean_length": 1004.69140625, + "completions/mean_terminated_length": 860.9467163085938, + "completions/min_length": 222.0, + "completions/min_terminated_length": 222.0, + "epoch": 0.08944269010838952, + "grad_norm": 0.15569855272769928, + "kl": 0.036041259765625, + "learning_rate": 8.938356164383561e-07, + "loss": 0.1649, + "num_tokens": 180502698.0, + "reward": 1.03076171875, + "reward_std": 0.3312453031539917, + "rewards/accuracy_reward/mean": 0.146484375, + "rewards/accuracy_reward/std": 0.35393697023391724, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.88427734375, + "rewards/tag_count_reward/std": 0.24312838912010193, + "step": 262 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 975.12109375, + "completions/mean_terminated_length": 864.1336059570312, + "completions/min_length": 154.0, + "completions/min_terminated_length": 154.0, + "epoch": 0.08978407442178032, + "grad_norm": 0.1347164362668991, + "kl": 0.03118896484375, + "learning_rate": 8.972602739726027e-07, + "loss": 0.1334, + "num_tokens": 181075736.0, + "reward": 1.05859375, + "reward_std": 0.30694493651390076, + "rewards/accuracy_reward/mean": 0.134765625, + "rewards/accuracy_reward/std": 0.3418070077896118, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.923828125, + "rewards/tag_count_reward/std": 0.2016531378030777, + "step": 263 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.146484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 1099.060546875, + "completions/mean_terminated_length": 936.1990356445312, + "completions/min_length": 150.0, + "completions/min_terminated_length": 150.0, + "epoch": 0.09012545873517112, + "grad_norm": 0.14101961255073547, + "kl": 0.032318115234375, + "learning_rate": 9.006849315068494e-07, + "loss": 0.1613, + "num_tokens": 181722023.0, + "reward": 0.96044921875, + "reward_std": 0.2774454951286316, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.88037109375, + "rewards/tag_count_reward/std": 0.2417331039905548, + "step": 264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.158203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 1101.751953125, + "completions/mean_terminated_length": 923.9187622070312, + "completions/min_length": 237.0, + "completions/min_terminated_length": 237.0, + "epoch": 0.09046684304856192, + "grad_norm": 0.14234143495559692, + "kl": 0.03485107421875, + "learning_rate": 9.041095890410958e-07, + "loss": 0.1435, + "num_tokens": 182367832.0, + "reward": 1.01416015625, + "reward_std": 0.2969135642051697, + "rewards/accuracy_reward/mean": 0.146484375, + "rewards/accuracy_reward/std": 0.35393697023391724, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.86767578125, + "rewards/tag_count_reward/std": 0.26207634806632996, + "step": 265 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.083984375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2041.0, + "completions/mean_length": 1116.673828125, + "completions/mean_terminated_length": 1031.2857666015625, + "completions/min_length": 275.0, + "completions/min_terminated_length": 275.0, + "epoch": 0.09080822736195272, + "grad_norm": 0.12336399406194687, + "kl": 0.026763916015625, + "learning_rate": 9.075342465753424e-07, + "loss": 0.1345, + "num_tokens": 183014945.0, + "reward": 1.05224609375, + "reward_std": 0.2811427116394043, + "rewards/accuracy_reward/mean": 0.1328125, + "rewards/accuracy_reward/std": 0.33970388770103455, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.91943359375, + "rewards/tag_count_reward/std": 0.19962850213050842, + "step": 266 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.130859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2023.0, + "completions/mean_length": 1091.169921875, + "completions/mean_terminated_length": 947.1078491210938, + "completions/min_length": 168.0, + "completions/min_terminated_length": 168.0, + "epoch": 0.09114961167534352, + "grad_norm": 0.12977954745292664, + "kl": 0.028839111328125, + "learning_rate": 9.10958904109589e-07, + "loss": 0.1588, + "num_tokens": 183651832.0, + "reward": 1.083984375, + "reward_std": 0.32412779331207275, + "rewards/accuracy_reward/mean": 0.19140625, + "rewards/accuracy_reward/std": 0.3937928080558777, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.892578125, + "rewards/tag_count_reward/std": 0.23602545261383057, + "step": 267 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.146484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2028.0, + "completions/mean_length": 1074.22265625, + "completions/mean_terminated_length": 907.0983276367188, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "epoch": 0.09149099598873432, + "grad_norm": 0.41138696670532227, + "kl": 0.034881591796875, + "learning_rate": 9.143835616438355e-07, + "loss": 0.1652, + "num_tokens": 184278586.0, + "reward": 1.01171875, + "reward_std": 0.2644849717617035, + "rewards/accuracy_reward/mean": 0.130859375, + "rewards/accuracy_reward/std": 0.33757632970809937, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.880859375, + "rewards/tag_count_reward/std": 0.24423858523368835, + "step": 268 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.146484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2013.0, + "completions/mean_length": 1102.029296875, + "completions/mean_terminated_length": 939.6773071289062, + "completions/min_length": 118.0, + "completions/min_terminated_length": 118.0, + "epoch": 0.09183238030212512, + "grad_norm": 0.13787169754505157, + "kl": 0.03399658203125, + "learning_rate": 9.178082191780822e-07, + "loss": 0.1477, + "num_tokens": 184915593.0, + "reward": 1.0498046875, + "reward_std": 0.36720407009124756, + "rewards/accuracy_reward/mean": 0.177734375, + "rewards/accuracy_reward/std": 0.3826628625392914, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8720703125, + "rewards/tag_count_reward/std": 0.25887611508369446, + "step": 269 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 1055.0078125, + "completions/mean_terminated_length": 942.7564697265625, + "completions/min_length": 155.0, + "completions/min_terminated_length": 155.0, + "epoch": 0.09217376461551592, + "grad_norm": 0.14218027889728546, + "kl": 0.03228759765625, + "learning_rate": 9.212328767123288e-07, + "loss": 0.1332, + "num_tokens": 185527117.0, + "reward": 1.08154296875, + "reward_std": 0.2797403633594513, + "rewards/accuracy_reward/mean": 0.171875, + "rewards/accuracy_reward/std": 0.3776407241821289, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.90966796875, + "rewards/tag_count_reward/std": 0.21676163375377655, + "step": 270 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.087890625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2030.0, + "completions/mean_length": 969.294921875, + "completions/mean_terminated_length": 865.3511962890625, + "completions/min_length": 70.0, + "completions/min_terminated_length": 70.0, + "epoch": 0.09251514892890672, + "grad_norm": 0.15893812477588654, + "kl": 0.031646728515625, + "learning_rate": 9.246575342465753e-07, + "loss": 0.1461, + "num_tokens": 186097316.0, + "reward": 1.1025390625, + "reward_std": 0.27628418803215027, + "rewards/accuracy_reward/mean": 0.1875, + "rewards/accuracy_reward/std": 0.39069411158561707, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9150390625, + "rewards/tag_count_reward/std": 0.20715762674808502, + "step": 271 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.095703125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1927.0, + "completions/mean_length": 973.080078125, + "completions/mean_terminated_length": 859.3196411132812, + "completions/min_length": 88.0, + "completions/min_terminated_length": 88.0, + "epoch": 0.09285653324229752, + "grad_norm": 0.17674598097801208, + "kl": 0.033203125, + "learning_rate": 9.280821917808219e-07, + "loss": 0.1953, + "num_tokens": 186669629.0, + "reward": 1.05517578125, + "reward_std": 0.2933085262775421, + "rewards/accuracy_reward/mean": 0.146484375, + "rewards/accuracy_reward/std": 0.35393697023391724, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.90869140625, + "rewards/tag_count_reward/std": 0.22027301788330078, + "step": 272 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14453125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 1073.92578125, + "completions/mean_terminated_length": 909.3561401367188, + "completions/min_length": 199.0, + "completions/min_terminated_length": 199.0, + "epoch": 0.09319791755568832, + "grad_norm": 0.6983135342597961, + "kl": 0.04931640625, + "learning_rate": 9.315068493150684e-07, + "loss": 0.1051, + "num_tokens": 187297543.0, + "reward": 1.0546875, + "reward_std": 0.3062615394592285, + "rewards/accuracy_reward/mean": 0.169921875, + "rewards/accuracy_reward/std": 0.3759314715862274, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.884765625, + "rewards/tag_count_reward/std": 0.24660581350326538, + "step": 273 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.107421875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 1065.720703125, + "completions/mean_terminated_length": 947.5032958984375, + "completions/min_length": 72.0, + "completions/min_terminated_length": 72.0, + "epoch": 0.09353930186907912, + "grad_norm": 0.14864228665828705, + "kl": 0.032867431640625, + "learning_rate": 9.349315068493149e-07, + "loss": 0.1027, + "num_tokens": 187914568.0, + "reward": 0.96630859375, + "reward_std": 0.25238245725631714, + "rewards/accuracy_reward/mean": 0.06451612710952759, + "rewards/accuracy_reward/std": 0.2459181249141693, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.90380859375, + "rewards/tag_count_reward/std": 0.22207027673721313, + "step": 274 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11328125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 1017.689453125, + "completions/mean_terminated_length": 886.0638427734375, + "completions/min_length": 83.0, + "completions/min_terminated_length": 83.0, + "epoch": 0.09388068618246992, + "grad_norm": 0.1409551501274109, + "kl": 0.031463623046875, + "learning_rate": 9.383561643835616e-07, + "loss": 0.1503, + "num_tokens": 188518681.0, + "reward": 1.00537109375, + "reward_std": 0.2579900920391083, + "rewards/accuracy_reward/mean": 0.11088709533214569, + "rewards/accuracy_reward/std": 0.3143092691898346, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.89794921875, + "rewards/tag_count_reward/std": 0.2318999469280243, + "step": 275 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.091796875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 1094.66015625, + "completions/mean_terminated_length": 998.3010864257812, + "completions/min_length": 218.0, + "completions/min_terminated_length": 218.0, + "epoch": 0.09422207049586072, + "grad_norm": 0.13870646059513092, + "kl": 0.03155517578125, + "learning_rate": 9.417808219178082e-07, + "loss": 0.114, + "num_tokens": 189151483.0, + "reward": 1.107421875, + "reward_std": 0.3174337148666382, + "rewards/accuracy_reward/mean": 0.185546875, + "rewards/accuracy_reward/std": 0.38912075757980347, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.921875, + "rewards/tag_count_reward/std": 0.20272120833396912, + "step": 276 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.185546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1997.0, + "completions/mean_length": 1112.8125, + "completions/mean_terminated_length": 899.7601928710938, + "completions/min_length": 167.0, + "completions/min_terminated_length": 167.0, + "epoch": 0.09456345480925152, + "grad_norm": 0.1267813742160797, + "kl": 0.033416748046875, + "learning_rate": 9.452054794520548e-07, + "loss": 0.1516, + "num_tokens": 189795563.0, + "reward": 0.9833984375, + "reward_std": 0.3141370415687561, + "rewards/accuracy_reward/mean": 0.13709677755832672, + "rewards/accuracy_reward/std": 0.34429675340652466, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8505859375, + "rewards/tag_count_reward/std": 0.27871251106262207, + "step": 277 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.189453125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 1171.390625, + "completions/mean_terminated_length": 966.4963989257812, + "completions/min_length": 181.0, + "completions/min_terminated_length": 181.0, + "epoch": 0.09490483912264232, + "grad_norm": 0.1538221538066864, + "kl": 0.034454345703125, + "learning_rate": 9.486301369863013e-07, + "loss": 0.19, + "num_tokens": 190471267.0, + "reward": 1.01171875, + "reward_std": 0.35299718379974365, + "rewards/accuracy_reward/mean": 0.16015625, + "rewards/accuracy_reward/std": 0.3671095669269562, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8515625, + "rewards/tag_count_reward/std": 0.27481982111930847, + "step": 278 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.158203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2041.0, + "completions/mean_length": 1120.703125, + "completions/mean_terminated_length": 946.4315185546875, + "completions/min_length": 148.0, + "completions/min_terminated_length": 148.0, + "epoch": 0.09524622343603312, + "grad_norm": 0.13736364245414734, + "kl": 0.03326416015625, + "learning_rate": 9.520547945205479e-07, + "loss": 0.1663, + "num_tokens": 191132507.0, + "reward": 1.0107421875, + "reward_std": 0.29221630096435547, + "rewards/accuracy_reward/mean": 0.14453125, + "rewards/accuracy_reward/std": 0.35197147727012634, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8662109375, + "rewards/tag_count_reward/std": 0.26342782378196716, + "step": 279 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2021.0, + "completions/mean_length": 1139.353515625, + "completions/mean_terminated_length": 980.965576171875, + "completions/min_length": 133.0, + "completions/min_terminated_length": 133.0, + "epoch": 0.09558760774942392, + "grad_norm": 0.14240288734436035, + "kl": 0.031890869140625, + "learning_rate": 9.554794520547946e-07, + "loss": 0.1552, + "num_tokens": 191784656.0, + "reward": 0.9638671875, + "reward_std": 0.33411771059036255, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8662109375, + "rewards/tag_count_reward/std": 0.26249760389328003, + "step": 280 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.154296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2031.0, + "completions/mean_length": 1133.05078125, + "completions/mean_terminated_length": 966.1200561523438, + "completions/min_length": 108.0, + "completions/min_terminated_length": 108.0, + "epoch": 0.09592899206281472, + "grad_norm": 0.14882326126098633, + "kl": 0.032806396484375, + "learning_rate": 9.58904109589041e-07, + "loss": 0.1661, + "num_tokens": 192439498.0, + "reward": 1.00732421875, + "reward_std": 0.3253936469554901, + "rewards/accuracy_reward/mean": 0.125, + "rewards/accuracy_reward/std": 0.3310423493385315, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.88232421875, + "rewards/tag_count_reward/std": 0.24469931423664093, + "step": 281 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.130859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2014.0, + "completions/mean_length": 1124.294921875, + "completions/mean_terminated_length": 985.22021484375, + "completions/min_length": 229.0, + "completions/min_terminated_length": 229.0, + "epoch": 0.09627037637620552, + "grad_norm": 0.1329856961965561, + "kl": 0.02911376953125, + "learning_rate": 9.623287671232875e-07, + "loss": 0.1325, + "num_tokens": 193085665.0, + "reward": 1.037109375, + "reward_std": 0.29661017656326294, + "rewards/accuracy_reward/mean": 0.150390625, + "rewards/accuracy_reward/std": 0.35780346393585205, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.88671875, + "rewards/tag_count_reward/std": 0.2389625608921051, + "step": 282 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1328125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 1120.37109375, + "completions/mean_terminated_length": 978.3018188476562, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "epoch": 0.09661176068959632, + "grad_norm": 0.1428883969783783, + "kl": 0.0323486328125, + "learning_rate": 9.657534246575343e-07, + "loss": 0.1881, + "num_tokens": 193747583.0, + "reward": 0.93994140625, + "reward_std": 0.2519282102584839, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.89111328125, + "rewards/tag_count_reward/std": 0.2324761003255844, + "step": 283 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1973.0, + "completions/mean_length": 1102.798828125, + "completions/mean_terminated_length": 938.0389404296875, + "completions/min_length": 199.0, + "completions/min_terminated_length": 199.0, + "epoch": 0.09695314500298712, + "grad_norm": 0.13743260502815247, + "kl": 0.030731201171875, + "learning_rate": 9.691780821917808e-07, + "loss": 0.1875, + "num_tokens": 194380648.0, + "reward": 1.044921875, + "reward_std": 0.3495681881904602, + "rewards/accuracy_reward/mean": 0.16796875, + "rewards/accuracy_reward/std": 0.374204158782959, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.876953125, + "rewards/tag_count_reward/std": 0.25459781289100647, + "step": 284 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 1021.318359375, + "completions/mean_terminated_length": 905.2586669921875, + "completions/min_length": 191.0, + "completions/min_terminated_length": 191.0, + "epoch": 0.09729452931637791, + "grad_norm": 0.15407976508140564, + "kl": 0.033599853515625, + "learning_rate": 9.726027397260274e-07, + "loss": 0.1479, + "num_tokens": 194986955.0, + "reward": 1.06396484375, + "reward_std": 0.2892252802848816, + "rewards/accuracy_reward/mean": 0.1640625, + "rewards/accuracy_reward/std": 0.37069445848464966, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.89990234375, + "rewards/tag_count_reward/std": 0.21471090614795685, + "step": 285 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.185546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2028.0, + "completions/mean_length": 1134.236328125, + "completions/mean_terminated_length": 926.0647583007812, + "completions/min_length": 172.0, + "completions/min_terminated_length": 172.0, + "epoch": 0.09763591362976871, + "grad_norm": 0.13862177729606628, + "kl": 0.032135009765625, + "learning_rate": 9.76027397260274e-07, + "loss": 0.1517, + "num_tokens": 195647028.0, + "reward": 0.99853515625, + "reward_std": 0.3387994170188904, + "rewards/accuracy_reward/mean": 0.142578125, + "rewards/accuracy_reward/std": 0.3499840497970581, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.85595703125, + "rewards/tag_count_reward/std": 0.27515992522239685, + "step": 286 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.111328125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 1058.318359375, + "completions/mean_terminated_length": 934.3363037109375, + "completions/min_length": 193.0, + "completions/min_terminated_length": 193.0, + "epoch": 0.09797729794315951, + "grad_norm": 0.14303427934646606, + "kl": 0.02935791015625, + "learning_rate": 9.794520547945205e-07, + "loss": 0.1308, + "num_tokens": 196275799.0, + "reward": 1.021484375, + "reward_std": 0.2921723425388336, + "rewards/accuracy_reward/mean": 0.111328125, + "rewards/accuracy_reward/std": 0.31484565138816833, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.91015625, + "rewards/tag_count_reward/std": 0.21892902255058289, + "step": 287 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11328125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2041.0, + "completions/mean_length": 1092.0546875, + "completions/mean_terminated_length": 969.9295043945312, + "completions/min_length": 197.0, + "completions/min_terminated_length": 197.0, + "epoch": 0.09831868225655031, + "grad_norm": 0.13289646804332733, + "kl": 0.031463623046875, + "learning_rate": 9.828767123287671e-07, + "loss": 0.1347, + "num_tokens": 196916243.0, + "reward": 0.990234375, + "reward_std": 0.2881425619125366, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.900390625, + "rewards/tag_count_reward/std": 0.22684305906295776, + "step": 288 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.099609375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2000.0, + "completions/mean_length": 1079.505859375, + "completions/mean_terminated_length": 972.3622436523438, + "completions/min_length": 147.0, + "completions/min_terminated_length": 147.0, + "epoch": 0.09866006656994111, + "grad_norm": 0.15402287244796753, + "kl": 0.032745361328125, + "learning_rate": 9.863013698630137e-07, + "loss": 0.0843, + "num_tokens": 197546662.0, + "reward": 1.0712890625, + "reward_std": 0.2916201651096344, + "rewards/accuracy_reward/mean": 0.15625, + "rewards/accuracy_reward/std": 0.36344730854034424, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9150390625, + "rewards/tag_count_reward/std": 0.2077472060918808, + "step": 289 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.103515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1942.0, + "completions/mean_length": 1006.572265625, + "completions/mean_terminated_length": 886.3202514648438, + "completions/min_length": 254.0, + "completions/min_terminated_length": 254.0, + "epoch": 0.09900145088333191, + "grad_norm": 0.1325167715549469, + "kl": 0.03338623046875, + "learning_rate": 9.897260273972602e-07, + "loss": 0.0719, + "num_tokens": 198137787.0, + "reward": 1.00390625, + "reward_std": 0.23973451554775238, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9140625, + "rewards/tag_count_reward/std": 0.20616121590137482, + "step": 290 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.177734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 1207.25390625, + "completions/mean_terminated_length": 1025.5250244140625, + "completions/min_length": 119.0, + "completions/min_terminated_length": 119.0, + "epoch": 0.09934283519672271, + "grad_norm": 325970.25, + "kl": 1808.0221862792969, + "learning_rate": 9.931506849315068e-07, + "loss": 72.6033, + "num_tokens": 198833341.0, + "reward": 0.98974609375, + "reward_std": 0.27682816982269287, + "rewards/accuracy_reward/mean": 0.140625, + "rewards/accuracy_reward/std": 0.3479743003845215, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.84912109375, + "rewards/tag_count_reward/std": 0.26692092418670654, + "step": 291 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1328125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1907.0, + "completions/mean_length": 1068.892578125, + "completions/mean_terminated_length": 918.939208984375, + "completions/min_length": 155.0, + "completions/min_terminated_length": 155.0, + "epoch": 0.09968421951011351, + "grad_norm": 0.16380666196346283, + "kl": 0.033355712890625, + "learning_rate": 9.965753424657534e-07, + "loss": 0.2172, + "num_tokens": 199456134.0, + "reward": 1.00732421875, + "reward_std": 0.31261372566223145, + "rewards/accuracy_reward/mean": 0.126953125, + "rewards/accuracy_reward/std": 0.33324605226516724, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.88037109375, + "rewards/tag_count_reward/std": 0.25261926651000977, + "step": 292 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.091796875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 1041.5, + "completions/mean_terminated_length": 939.7677612304688, + "completions/min_length": 247.0, + "completions/min_terminated_length": 247.0, + "epoch": 0.10002560382350431, + "grad_norm": 2.1582608222961426, + "kl": 0.0596923828125, + "learning_rate": 1e-06, + "loss": 0.1275, + "num_tokens": 200065414.0, + "reward": 1.09130859375, + "reward_std": 0.3151357173919678, + "rewards/accuracy_reward/mean": 0.177734375, + "rewards/accuracy_reward/std": 0.3826628625392914, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.91357421875, + "rewards/tag_count_reward/std": 0.20978116989135742, + "step": 293 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.111328125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 1068.361328125, + "completions/mean_terminated_length": 945.6373901367188, + "completions/min_length": 154.0, + "completions/min_terminated_length": 154.0, + "epoch": 0.10036698813689511, + "grad_norm": 0.18343022465705872, + "kl": 0.035888671875, + "learning_rate": 9.99999680653653e-07, + "loss": 0.1589, + "num_tokens": 200689711.0, + "reward": 0.9853515625, + "reward_std": 0.2748567759990692, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8955078125, + "rewards/tag_count_reward/std": 0.2326553463935852, + "step": 294 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.123046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 1127.794921875, + "completions/mean_terminated_length": 998.6793212890625, + "completions/min_length": 188.0, + "completions/min_terminated_length": 188.0, + "epoch": 0.10070837245028591, + "grad_norm": 0.12994924187660217, + "kl": 0.03369140625, + "learning_rate": 9.999987226150655e-07, + "loss": 0.13, + "num_tokens": 201340918.0, + "reward": 0.9599609375, + "reward_std": 0.25544899702072144, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8935546875, + "rewards/tag_count_reward/std": 0.23750056326389313, + "step": 295 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11328125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 1050.5390625, + "completions/mean_terminated_length": 923.110107421875, + "completions/min_length": 188.0, + "completions/min_terminated_length": 188.0, + "epoch": 0.10104975676367671, + "grad_norm": 0.1931072175502777, + "kl": 0.032958984375, + "learning_rate": 9.99997125885597e-07, + "loss": 0.1366, + "num_tokens": 201950554.0, + "reward": 1.06298828125, + "reward_std": 0.2893334925174713, + "rewards/accuracy_reward/mean": 0.15625, + "rewards/accuracy_reward/std": 0.36344730854034424, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.90673828125, + "rewards/tag_count_reward/std": 0.22386592626571655, + "step": 296 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14453125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2017.0, + "completions/mean_length": 1171.7890625, + "completions/mean_terminated_length": 1023.7533569335938, + "completions/min_length": 248.0, + "completions/min_terminated_length": 248.0, + "epoch": 0.10139114107706751, + "grad_norm": 0.1361197531223297, + "kl": 0.031890869140625, + "learning_rate": 9.99994890467514e-07, + "loss": 0.1594, + "num_tokens": 202628494.0, + "reward": 1.037109375, + "reward_std": 0.32598963379859924, + "rewards/accuracy_reward/mean": 0.14453125, + "rewards/accuracy_reward/std": 0.35197147727012634, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.892578125, + "rewards/tag_count_reward/std": 0.24216407537460327, + "step": 297 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 1204.568359375, + "completions/mean_terminated_length": 1057.548095703125, + "completions/min_length": 214.0, + "completions/min_terminated_length": 214.0, + "epoch": 0.10173252539045831, + "grad_norm": 0.12307877838611603, + "kl": 0.03009033203125, + "learning_rate": 9.999920163639891e-07, + "loss": 0.1402, + "num_tokens": 203318801.0, + "reward": 1.0185546875, + "reward_std": 0.3293812870979309, + "rewards/accuracy_reward/mean": 0.13671875, + "rewards/accuracy_reward/std": 0.3438861668109894, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8818359375, + "rewards/tag_count_reward/std": 0.2422015517950058, + "step": 298 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.169921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2029.0, + "completions/mean_length": 1149.375, + "completions/mean_terminated_length": 965.421142578125, + "completions/min_length": 237.0, + "completions/min_terminated_length": 237.0, + "epoch": 0.10207390970384911, + "grad_norm": 0.13443991541862488, + "kl": 0.03179931640625, + "learning_rate": 9.999885035791019e-07, + "loss": 0.1773, + "num_tokens": 203987681.0, + "reward": 1.0078125, + "reward_std": 0.3582117259502411, + "rewards/accuracy_reward/mean": 0.15234375, + "rewards/accuracy_reward/std": 0.35970520973205566, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.85546875, + "rewards/tag_count_reward/std": 0.27064353227615356, + "step": 299 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.146484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 1174.439453125, + "completions/mean_terminated_length": 1024.5147705078125, + "completions/min_length": 209.0, + "completions/min_terminated_length": 209.0, + "epoch": 0.10241529401723991, + "grad_norm": 0.1388065218925476, + "kl": 0.0328369140625, + "learning_rate": 9.999843521178375e-07, + "loss": 0.1374, + "num_tokens": 204670050.0, + "reward": 0.9462890625, + "reward_std": 0.2924199104309082, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8701171875, + "rewards/tag_count_reward/std": 0.2564731240272522, + "step": 300 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 1104.498046875, + "completions/mean_terminated_length": 979.25439453125, + "completions/min_length": 133.0, + "completions/min_terminated_length": 133.0, + "epoch": 0.10275667833063071, + "grad_norm": 0.12873072922229767, + "kl": 0.031036376953125, + "learning_rate": 9.99979561986089e-07, + "loss": 0.1286, + "num_tokens": 205310001.0, + "reward": 1.091796875, + "reward_std": 0.3492361605167389, + "rewards/accuracy_reward/mean": 0.185546875, + "rewards/accuracy_reward/std": 0.38912075757980347, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.90625, + "rewards/tag_count_reward/std": 0.22502446174621582, + "step": 301 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.130859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 1101.455078125, + "completions/mean_terminated_length": 958.9415893554688, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "epoch": 0.10309806264402151, + "grad_norm": 0.13499902188777924, + "kl": 0.03302001953125, + "learning_rate": 9.999741331906542e-07, + "loss": 0.1448, + "num_tokens": 205951834.0, + "reward": 1.00927734375, + "reward_std": 0.28933781385421753, + "rewards/accuracy_reward/mean": 0.123046875, + "rewards/accuracy_reward/std": 0.32881227135658264, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.88623046875, + "rewards/tag_count_reward/std": 0.23949728906154633, + "step": 302 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1328125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 1156.25390625, + "completions/mean_terminated_length": 1019.68017578125, + "completions/min_length": 146.0, + "completions/min_terminated_length": 146.0, + "epoch": 0.1034394469574123, + "grad_norm": 0.1338455229997635, + "kl": 0.0301513671875, + "learning_rate": 9.99968065739239e-07, + "loss": 0.1571, + "num_tokens": 206628780.0, + "reward": 1.00341796875, + "reward_std": 0.2902722954750061, + "rewards/accuracy_reward/mean": 0.1171875, + "rewards/accuracy_reward/std": 0.32195815443992615, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.88623046875, + "rewards/tag_count_reward/std": 0.23485609889030457, + "step": 303 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14453125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 1173.71875, + "completions/mean_terminated_length": 1026.009033203125, + "completions/min_length": 196.0, + "completions/min_terminated_length": 196.0, + "epoch": 0.1037808312708031, + "grad_norm": 0.1281381994485855, + "kl": 0.029388427734375, + "learning_rate": 9.999613596404544e-07, + "loss": 0.1509, + "num_tokens": 207305836.0, + "reward": 1.00146484375, + "reward_std": 0.3014447093009949, + "rewards/accuracy_reward/mean": 0.11328125, + "rewards/accuracy_reward/std": 0.3172462284564972, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.88818359375, + "rewards/tag_count_reward/std": 0.2399078905582428, + "step": 304 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.154296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 1184.798828125, + "completions/mean_terminated_length": 1027.3094482421875, + "completions/min_length": 220.0, + "completions/min_terminated_length": 220.0, + "epoch": 0.1041222155841939, + "grad_norm": 0.10877677798271179, + "kl": 0.028076171875, + "learning_rate": 9.999540149038193e-07, + "loss": 0.0941, + "num_tokens": 207985909.0, + "reward": 1.03955078125, + "reward_std": 0.26682427525520325, + "rewards/accuracy_reward/mean": 0.150390625, + "rewards/accuracy_reward/std": 0.35780346393585205, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.88916015625, + "rewards/tag_count_reward/std": 0.24339549243450165, + "step": 305 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.103515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 1093.181640625, + "completions/mean_terminated_length": 982.9302368164062, + "completions/min_length": 195.0, + "completions/min_terminated_length": 195.0, + "epoch": 0.1044635998975847, + "grad_norm": 0.13633041083812714, + "kl": 0.03131103515625, + "learning_rate": 9.999460315397577e-07, + "loss": 0.1139, + "num_tokens": 208629042.0, + "reward": 1.0380859375, + "reward_std": 0.27025923132896423, + "rewards/accuracy_reward/mean": 0.134765625, + "rewards/accuracy_reward/std": 0.3418070077896118, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9033203125, + "rewards/tag_count_reward/std": 0.2226831316947937, + "step": 306 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.162109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2013.0, + "completions/mean_length": 1133.87890625, + "completions/mean_terminated_length": 957.02099609375, + "completions/min_length": 110.0, + "completions/min_terminated_length": 110.0, + "epoch": 0.1048049842109755, + "grad_norm": 0.12762504816055298, + "kl": 0.03314208984375, + "learning_rate": 9.999374095596004e-07, + "loss": 0.1499, + "num_tokens": 209281860.0, + "reward": 0.970703125, + "reward_std": 0.27441883087158203, + "rewards/accuracy_reward/mean": 0.10000000149011612, + "rewards/accuracy_reward/std": 0.30031299591064453, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.876953125, + "rewards/tag_count_reward/std": 0.25266891717910767, + "step": 307 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.130859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 1178.130859375, + "completions/mean_terminated_length": 1047.161865234375, + "completions/min_length": 190.0, + "completions/min_terminated_length": 190.0, + "epoch": 0.1051463685243663, + "grad_norm": 0.12441974133253098, + "kl": 0.03045654296875, + "learning_rate": 9.99928148975585e-07, + "loss": 0.1385, + "num_tokens": 209965031.0, + "reward": 1.0419921875, + "reward_std": 0.27181071043014526, + "rewards/accuracy_reward/mean": 0.142578125, + "rewards/accuracy_reward/std": 0.3499840497970581, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8994140625, + "rewards/tag_count_reward/std": 0.22695045173168182, + "step": 308 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08984375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2022.0, + "completions/mean_length": 1064.421875, + "completions/mean_terminated_length": 967.3304443359375, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "epoch": 0.1054877528377571, + "grad_norm": 0.1263512820005417, + "kl": 0.030517578125, + "learning_rate": 9.99918249800855e-07, + "loss": 0.1083, + "num_tokens": 210586767.0, + "reward": 1.08251953125, + "reward_std": 0.2911805808544159, + "rewards/accuracy_reward/mean": 0.166015625, + "rewards/accuracy_reward/std": 0.3724585771560669, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.91650390625, + "rewards/tag_count_reward/std": 0.20804768800735474, + "step": 309 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 1017.6171875, + "completions/mean_terminated_length": 930.296630859375, + "completions/min_length": 178.0, + "completions/min_terminated_length": 178.0, + "epoch": 0.1058291371511479, + "grad_norm": 0.1431308537721634, + "kl": 0.0330810546875, + "learning_rate": 9.999077120494608e-07, + "loss": 0.1317, + "num_tokens": 211180107.0, + "reward": 1.08154296875, + "reward_std": 0.2786792814731598, + "rewards/accuracy_reward/mean": 0.16015625, + "rewards/accuracy_reward/std": 0.3671095669269562, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.92138671875, + "rewards/tag_count_reward/std": 0.2040361613035202, + "step": 310 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.115234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1982.0, + "completions/mean_length": 1058.0546875, + "completions/mean_terminated_length": 929.1213989257812, + "completions/min_length": 162.0, + "completions/min_terminated_length": 162.0, + "epoch": 0.1061705214645387, + "grad_norm": 0.14184510707855225, + "kl": 0.03277587890625, + "learning_rate": 9.998965357363583e-07, + "loss": 0.1255, + "num_tokens": 211800999.0, + "reward": 0.97998046875, + "reward_std": 0.2651514410972595, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.90185546875, + "rewards/tag_count_reward/std": 0.22504940629005432, + "step": 311 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.185546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 1193.66796875, + "completions/mean_terminated_length": 999.0360107421875, + "completions/min_length": 198.0, + "completions/min_terminated_length": 198.0, + "epoch": 0.1065119057779295, + "grad_norm": 0.127924844622612, + "kl": 0.03619384765625, + "learning_rate": 9.998847208774107e-07, + "loss": 0.1751, + "num_tokens": 212486701.0, + "reward": 0.9443359375, + "reward_std": 0.3127998411655426, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8525390625, + "rewards/tag_count_reward/std": 0.2726678252220154, + "step": 312 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.146484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 1163.001953125, + "completions/mean_terminated_length": 1011.1143798828125, + "completions/min_length": 194.0, + "completions/min_terminated_length": 194.0, + "epoch": 0.1068532900913203, + "grad_norm": 0.12907107174396515, + "kl": 0.031494140625, + "learning_rate": 9.998722674893869e-07, + "loss": 0.1676, + "num_tokens": 213154302.0, + "reward": 1.0478515625, + "reward_std": 0.3261502981185913, + "rewards/accuracy_reward/mean": 0.16015625, + "rewards/accuracy_reward/std": 0.3671095669269562, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8876953125, + "rewards/tag_count_reward/std": 0.24095164239406586, + "step": 313 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2029.0, + "completions/mean_length": 1181.607421875, + "completions/mean_terminated_length": 1044.39599609375, + "completions/min_length": 178.0, + "completions/min_terminated_length": 178.0, + "epoch": 0.1071946744047111, + "grad_norm": 0.12877912819385529, + "kl": 0.0335693359375, + "learning_rate": 9.99859175589962e-07, + "loss": 0.1381, + "num_tokens": 213838085.0, + "reward": 1.015625, + "reward_std": 0.317222535610199, + "rewards/accuracy_reward/mean": 0.126953125, + "rewards/accuracy_reward/std": 0.33324605226516724, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.888671875, + "rewards/tag_count_reward/std": 0.23885856568813324, + "step": 314 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.142578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 1161.228515625, + "completions/mean_terminated_length": 1013.7699584960938, + "completions/min_length": 112.0, + "completions/min_terminated_length": 112.0, + "epoch": 0.1075360587181019, + "grad_norm": 0.2052370309829712, + "kl": 0.034515380859375, + "learning_rate": 9.998454451977178e-07, + "loss": 0.1532, + "num_tokens": 214508122.0, + "reward": 0.9892578125, + "reward_std": 0.29635995626449585, + "rewards/accuracy_reward/mean": 0.10080645233392715, + "rewards/accuracy_reward/std": 0.30137622356414795, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8916015625, + "rewards/tag_count_reward/std": 0.2345370054244995, + "step": 315 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14453125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2029.0, + "completions/mean_length": 1156.36328125, + "completions/mean_terminated_length": 1005.721435546875, + "completions/min_length": 147.0, + "completions/min_terminated_length": 147.0, + "epoch": 0.1078774430314927, + "grad_norm": 0.1357164978981018, + "kl": 0.03118896484375, + "learning_rate": 9.99831076332142e-07, + "loss": 0.2021, + "num_tokens": 215176228.0, + "reward": 0.96875, + "reward_std": 0.27862483263015747, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.888671875, + "rewards/tag_count_reward/std": 0.2419114112854004, + "step": 316 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1328125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 1110.935546875, + "completions/mean_terminated_length": 967.4212036132812, + "completions/min_length": 201.0, + "completions/min_terminated_length": 201.0, + "epoch": 0.1082188273448835, + "grad_norm": 0.7026339173316956, + "kl": 0.042694091796875, + "learning_rate": 9.998160690136289e-07, + "loss": 0.133, + "num_tokens": 215820387.0, + "reward": 1.03515625, + "reward_std": 0.31893688440322876, + "rewards/accuracy_reward/mean": 0.13671875, + "rewards/accuracy_reward/std": 0.3438861668109894, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8984375, + "rewards/tag_count_reward/std": 0.2318510115146637, + "step": 317 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08984375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1983.0, + "completions/mean_length": 938.87109375, + "completions/mean_terminated_length": 829.3862915039062, + "completions/min_length": 130.0, + "completions/min_terminated_length": 130.0, + "epoch": 0.1085602116582743, + "grad_norm": 5.583955764770508, + "kl": 0.05126953125, + "learning_rate": 9.998004232634777e-07, + "loss": 0.1829, + "num_tokens": 216372337.0, + "reward": 1.052734375, + "reward_std": 0.2629846930503845, + "rewards/accuracy_reward/mean": 0.13104838132858276, + "rewards/accuracy_reward/std": 0.3377939760684967, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.92578125, + "rewards/tag_count_reward/std": 0.20238155126571655, + "step": 318 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.138671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 1135.80078125, + "completions/mean_terminated_length": 988.9387817382812, + "completions/min_length": 210.0, + "completions/min_terminated_length": 210.0, + "epoch": 0.1089015959716651, + "grad_norm": 0.16040125489234924, + "kl": 0.03277587890625, + "learning_rate": 9.997841391038957e-07, + "loss": 0.1377, + "num_tokens": 217024795.0, + "reward": 1.017578125, + "reward_std": 0.31150808930397034, + "rewards/accuracy_reward/mean": 0.13508065044879913, + "rewards/accuracy_reward/std": 0.3421548008918762, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.88671875, + "rewards/tag_count_reward/std": 0.2410012036561966, + "step": 319 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 1056.83984375, + "completions/mean_terminated_length": 972.8432006835938, + "completions/min_length": 176.0, + "completions/min_terminated_length": 176.0, + "epoch": 0.1092429802850559, + "grad_norm": 0.13190127909183502, + "kl": 0.03271484375, + "learning_rate": 9.997672165579948e-07, + "loss": 0.1314, + "num_tokens": 217648313.0, + "reward": 0.99755859375, + "reward_std": 0.2191423624753952, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.92138671875, + "rewards/tag_count_reward/std": 0.20040719211101532, + "step": 320 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.134765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 1155.67578125, + "completions/mean_terminated_length": 1016.6907958984375, + "completions/min_length": 171.0, + "completions/min_terminated_length": 171.0, + "epoch": 0.1095843645984467, + "grad_norm": 0.12830433249473572, + "kl": 0.03466796875, + "learning_rate": 9.997496556497934e-07, + "loss": 0.1481, + "num_tokens": 218315059.0, + "reward": 1.048828125, + "reward_std": 0.3038339912891388, + "rewards/accuracy_reward/mean": 0.150390625, + "rewards/accuracy_reward/std": 0.35780346393585205, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8984375, + "rewards/tag_count_reward/std": 0.2344738394021988, + "step": 321 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.087890625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2022.0, + "completions/mean_length": 1029.49609375, + "completions/mean_terminated_length": 931.3533325195312, + "completions/min_length": 267.0, + "completions/min_terminated_length": 267.0, + "epoch": 0.1099257489118375, + "grad_norm": 0.24386551976203918, + "kl": 0.03375244140625, + "learning_rate": 9.997314564042165e-07, + "loss": 0.1044, + "num_tokens": 218913713.0, + "reward": 1.0634765625, + "reward_std": 0.2716171443462372, + "rewards/accuracy_reward/mean": 0.14453125, + "rewards/accuracy_reward/std": 0.35197147727012634, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9189453125, + "rewards/tag_count_reward/std": 0.2015654444694519, + "step": 322 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12890625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 1173.41796875, + "completions/mean_terminated_length": 1043.99560546875, + "completions/min_length": 218.0, + "completions/min_terminated_length": 218.0, + "epoch": 0.1102671332252283, + "grad_norm": 0.13542306423187256, + "kl": 0.03369140625, + "learning_rate": 9.997126188470941e-07, + "loss": 0.1083, + "num_tokens": 219593111.0, + "reward": 1.0234375, + "reward_std": 0.28221234679222107, + "rewards/accuracy_reward/mean": 0.123046875, + "rewards/accuracy_reward/std": 0.32881227135658264, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.900390625, + "rewards/tag_count_reward/std": 0.22952312231063843, + "step": 323 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.138671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 1177.892578125, + "completions/mean_terminated_length": 1037.8072509765625, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "epoch": 0.1106085175386191, + "grad_norm": 0.20564241707324982, + "kl": 0.035614013671875, + "learning_rate": 9.996931430051626e-07, + "loss": 0.1422, + "num_tokens": 220286640.0, + "reward": 0.95263671875, + "reward_std": 0.2707866132259369, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.89013671875, + "rewards/tag_count_reward/std": 0.2497853934764862, + "step": 324 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07421875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 1053.9921875, + "completions/mean_terminated_length": 974.3037719726562, + "completions/min_length": 161.0, + "completions/min_terminated_length": 161.0, + "epoch": 0.1109499018520099, + "grad_norm": 0.15150968730449677, + "kl": 0.036376953125, + "learning_rate": 9.99673028906065e-07, + "loss": 0.1305, + "num_tokens": 220895260.0, + "reward": 1.10986328125, + "reward_std": 0.28572919964790344, + "rewards/accuracy_reward/mean": 0.177734375, + "rewards/accuracy_reward/std": 0.3826628625392914, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.93212890625, + "rewards/tag_count_reward/std": 0.18809467554092407, + "step": 325 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.123046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 1158.326171875, + "completions/mean_terminated_length": 1033.4945068359375, + "completions/min_length": 183.0, + "completions/min_terminated_length": 183.0, + "epoch": 0.1112912861654007, + "grad_norm": 0.14217494428157806, + "kl": 0.032989501953125, + "learning_rate": 9.996522765783488e-07, + "loss": 0.1176, + "num_tokens": 221562723.0, + "reward": 1.03955078125, + "reward_std": 0.2918170392513275, + "rewards/accuracy_reward/mean": 0.13671875, + "rewards/accuracy_reward/std": 0.3438861668109894, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.90283203125, + "rewards/tag_count_reward/std": 0.22219502925872803, + "step": 326 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 1088.98046875, + "completions/mean_terminated_length": 998.8162841796875, + "completions/min_length": 236.0, + "completions/min_terminated_length": 236.0, + "epoch": 0.1116326704787915, + "grad_norm": 0.28022122383117676, + "kl": 0.033935546875, + "learning_rate": 9.996308860514686e-07, + "loss": 0.0746, + "num_tokens": 222194681.0, + "reward": 1.142578125, + "reward_std": 0.33608290553092957, + "rewards/accuracy_reward/mean": 0.21484375, + "rewards/accuracy_reward/std": 0.4111155867576599, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.927734375, + "rewards/tag_count_reward/std": 0.1880800724029541, + "step": 327 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.111328125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 1109.802734375, + "completions/mean_terminated_length": 992.2703857421875, + "completions/min_length": 216.0, + "completions/min_terminated_length": 216.0, + "epoch": 0.1119740547921823, + "grad_norm": 0.18140637874603271, + "kl": 0.03533935546875, + "learning_rate": 9.996088573557843e-07, + "loss": 0.1316, + "num_tokens": 222838068.0, + "reward": 0.9814453125, + "reward_std": 0.2408694475889206, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9072265625, + "rewards/tag_count_reward/std": 0.22542962431907654, + "step": 328 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.099609375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 1082.96484375, + "completions/mean_terminated_length": 976.2039184570312, + "completions/min_length": 168.0, + "completions/min_terminated_length": 168.0, + "epoch": 0.1123154391055731, + "grad_norm": 0.13812144100666046, + "kl": 0.03448486328125, + "learning_rate": 9.995861905225617e-07, + "loss": 0.1212, + "num_tokens": 223464770.0, + "reward": 1.1123046875, + "reward_std": 0.27613845467567444, + "rewards/accuracy_reward/mean": 0.19140625, + "rewards/accuracy_reward/std": 0.3937928080558777, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9208984375, + "rewards/tag_count_reward/std": 0.20474500954151154, + "step": 329 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14453125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2011.0, + "completions/mean_length": 1178.630859375, + "completions/mean_terminated_length": 1031.7510986328125, + "completions/min_length": 166.0, + "completions/min_terminated_length": 166.0, + "epoch": 0.1126568234189639, + "grad_norm": 0.3180902898311615, + "kl": 0.03668212890625, + "learning_rate": 9.995628855839721e-07, + "loss": 0.1542, + "num_tokens": 224145125.0, + "reward": 0.95361328125, + "reward_std": 0.2629111409187317, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.88134765625, + "rewards/tag_count_reward/std": 0.23865395784378052, + "step": 330 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.130859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2033.0, + "completions/mean_length": 1097.04296875, + "completions/mean_terminated_length": 953.8651733398438, + "completions/min_length": 234.0, + "completions/min_terminated_length": 234.0, + "epoch": 0.1129982077323547, + "grad_norm": 0.41690030694007874, + "kl": 0.04473876953125, + "learning_rate": 9.995389425730923e-07, + "loss": 0.1594, + "num_tokens": 224789451.0, + "reward": 0.98193359375, + "reward_std": 0.287788987159729, + "rewards/accuracy_reward/mean": 0.087890625, + "rewards/accuracy_reward/std": 0.2834126651287079, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.89404296875, + "rewards/tag_count_reward/std": 0.2343510389328003, + "step": 331 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 1192.52734375, + "completions/mean_terminated_length": 1014.9764404296875, + "completions/min_length": 178.0, + "completions/min_terminated_length": 178.0, + "epoch": 0.1133395920457455, + "grad_norm": 59.31746292114258, + "kl": 0.782470703125, + "learning_rate": 9.995143615239056e-07, + "loss": 0.2046, + "num_tokens": 225479369.0, + "reward": 0.97314453125, + "reward_std": 0.2881585955619812, + "rewards/accuracy_reward/mean": 0.103515625, + "rewards/accuracy_reward/std": 0.30492907762527466, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.86962890625, + "rewards/tag_count_reward/std": 0.2583639323711395, + "step": 332 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.095703125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1999.0, + "completions/mean_length": 1073.884765625, + "completions/mean_terminated_length": 970.7926025390625, + "completions/min_length": 204.0, + "completions/min_terminated_length": 204.0, + "epoch": 0.1136809763591363, + "grad_norm": 0.3465381860733032, + "kl": 0.046630859375, + "learning_rate": 9.994891424712998e-07, + "loss": 0.1354, + "num_tokens": 226107262.0, + "reward": 1.03369140625, + "reward_std": 0.2943110466003418, + "rewards/accuracy_reward/mean": 0.115234375, + "rewards/accuracy_reward/std": 0.3196168541908264, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.91845703125, + "rewards/tag_count_reward/std": 0.21345669031143188, + "step": 333 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.095703125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2009.0, + "completions/mean_length": 1059.50390625, + "completions/mean_terminated_length": 954.8898315429688, + "completions/min_length": 209.0, + "completions/min_terminated_length": 209.0, + "epoch": 0.11402236067252709, + "grad_norm": 0.44690296053886414, + "kl": 0.0445556640625, + "learning_rate": 9.99463285451069e-07, + "loss": 0.1222, + "num_tokens": 226726960.0, + "reward": 1.07568359375, + "reward_std": 0.2955414354801178, + "rewards/accuracy_reward/mean": 0.146484375, + "rewards/accuracy_reward/std": 0.35393697023391724, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.92919921875, + "rewards/tag_count_reward/std": 0.19280590116977692, + "step": 334 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.087890625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 1074.240234375, + "completions/mean_terminated_length": 980.4089965820312, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.11436374498591789, + "grad_norm": 0.4894280433654785, + "kl": 0.044921875, + "learning_rate": 9.994367904999127e-07, + "loss": 0.1044, + "num_tokens": 227353979.0, + "reward": 1.08251953125, + "reward_std": 0.3042677938938141, + "rewards/accuracy_reward/mean": 0.16015625, + "rewards/accuracy_reward/std": 0.3671095669269562, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.92236328125, + "rewards/tag_count_reward/std": 0.2020028978586197, + "step": 335 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1328125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 1173.904296875, + "completions/mean_terminated_length": 1040.0338134765625, + "completions/min_length": 255.0, + "completions/min_terminated_length": 255.0, + "epoch": 0.11470512929930869, + "grad_norm": 15565.7607421875, + "kl": 145.09112548828125, + "learning_rate": 9.994096576554353e-07, + "loss": 5.8975, + "num_tokens": 228028906.0, + "reward": 1.11474609375, + "reward_std": 0.34920042753219604, + "rewards/accuracy_reward/mean": 0.216796875, + "rewards/accuracy_reward/std": 0.4124660789966583, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.89794921875, + "rewards/tag_count_reward/std": 0.24171333014965057, + "step": 336 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.142578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 1165.974609375, + "completions/mean_terminated_length": 1019.3052978515625, + "completions/min_length": 236.0, + "completions/min_terminated_length": 236.0, + "epoch": 0.11504651361269949, + "grad_norm": 3.400146245956421, + "kl": 0.08160400390625, + "learning_rate": 9.993818869561467e-07, + "loss": 0.1191, + "num_tokens": 228702605.0, + "reward": 1.0546875, + "reward_std": 0.30855315923690796, + "rewards/accuracy_reward/mean": 0.17540322244167328, + "rewards/accuracy_reward/std": 0.3806955814361572, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.884765625, + "rewards/tag_count_reward/std": 0.2380250245332718, + "step": 337 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.140625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2028.0, + "completions/mean_length": 1144.80859375, + "completions/mean_terminated_length": 997.0136108398438, + "completions/min_length": 173.0, + "completions/min_terminated_length": 173.0, + "epoch": 0.11538789792609029, + "grad_norm": 0.46814846992492676, + "kl": 0.04425048828125, + "learning_rate": 9.99353478441463e-07, + "loss": 0.1017, + "num_tokens": 229372635.0, + "reward": 1.00732421875, + "reward_std": 0.26241379976272583, + "rewards/accuracy_reward/mean": 0.115234375, + "rewards/accuracy_reward/std": 0.3196168541908264, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.89208984375, + "rewards/tag_count_reward/std": 0.23914992809295654, + "step": 338 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.126953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 1107.2265625, + "completions/mean_terminated_length": 970.425048828125, + "completions/min_length": 296.0, + "completions/min_terminated_length": 296.0, + "epoch": 0.11572928223948109, + "grad_norm": 1.4173657894134521, + "kl": 0.06671142578125, + "learning_rate": 9.993244321517045e-07, + "loss": 0.1712, + "num_tokens": 230021071.0, + "reward": 1.05712890625, + "reward_std": 0.35515138506889343, + "rewards/accuracy_reward/mean": 0.154296875, + "rewards/accuracy_reward/std": 0.36158639192581177, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.90283203125, + "rewards/tag_count_reward/std": 0.23503504693508148, + "step": 339 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.115234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1994.0, + "completions/mean_length": 1103.939453125, + "completions/mean_terminated_length": 980.9823608398438, + "completions/min_length": 242.0, + "completions/min_terminated_length": 242.0, + "epoch": 0.11607066655287189, + "grad_norm": 0.6045756936073303, + "kl": 0.04412841796875, + "learning_rate": 9.99294748128097e-07, + "loss": 0.1306, + "num_tokens": 230658224.0, + "reward": 1.015625, + "reward_std": 0.25914958119392395, + "rewards/accuracy_reward/mean": 0.11491935700178146, + "rewards/accuracy_reward/std": 0.3192465901374817, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.904296875, + "rewards/tag_count_reward/std": 0.22310540080070496, + "step": 340 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 1199.4140625, + "completions/mean_terminated_length": 1003.5865478515625, + "completions/min_length": 217.0, + "completions/min_terminated_length": 217.0, + "epoch": 0.11641205086626269, + "grad_norm": 0.34343641996383667, + "kl": 0.0516357421875, + "learning_rate": 9.992644264127717e-07, + "loss": 0.1072, + "num_tokens": 231350276.0, + "reward": 1.06982421875, + "reward_std": 0.3374538719654083, + "rewards/accuracy_reward/mean": 0.21484375, + "rewards/accuracy_reward/std": 0.4111155867576599, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.85498046875, + "rewards/tag_count_reward/std": 0.2759782373905182, + "step": 341 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12890625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 1096.859375, + "completions/mean_terminated_length": 956.107666015625, + "completions/min_length": 228.0, + "completions/min_terminated_length": 228.0, + "epoch": 0.11675343517965349, + "grad_norm": 0.5945049524307251, + "kl": 0.03814697265625, + "learning_rate": 9.992334670487646e-07, + "loss": 0.1329, + "num_tokens": 231985548.0, + "reward": 1.0166015625, + "reward_std": 0.2711396813392639, + "rewards/accuracy_reward/mean": 0.11290322244167328, + "rewards/accuracy_reward/std": 0.3167939782142639, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9072265625, + "rewards/tag_count_reward/std": 0.21993711590766907, + "step": 342 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 1054.84375, + "completions/mean_terminated_length": 932.877197265625, + "completions/min_length": 226.0, + "completions/min_terminated_length": 226.0, + "epoch": 0.11709481949304429, + "grad_norm": 2.2203285694122314, + "kl": 0.07745361328125, + "learning_rate": 9.99201870080017e-07, + "loss": 0.1794, + "num_tokens": 232594364.0, + "reward": 0.99560546875, + "reward_std": 0.24448131024837494, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.91162109375, + "rewards/tag_count_reward/std": 0.21587374806404114, + "step": 343 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.146484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2004.0, + "completions/mean_length": 1219.873046875, + "completions/mean_terminated_length": 1077.7459716796875, + "completions/min_length": 204.0, + "completions/min_terminated_length": 204.0, + "epoch": 0.11743620380643509, + "grad_norm": 0.4197451174259186, + "kl": 0.04046630859375, + "learning_rate": 9.99169635551375e-07, + "loss": 0.1429, + "num_tokens": 233295003.0, + "reward": 0.9736328125, + "reward_std": 0.26731252670288086, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8876953125, + "rewards/tag_count_reward/std": 0.2394239604473114, + "step": 344 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08984375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 1111.126953125, + "completions/mean_terminated_length": 1018.6459350585938, + "completions/min_length": 184.0, + "completions/min_terminated_length": 184.0, + "epoch": 0.11777758811982589, + "grad_norm": 0.35124969482421875, + "kl": 0.044921875, + "learning_rate": 9.991367635085897e-07, + "loss": 0.1177, + "num_tokens": 233943052.0, + "reward": 1.0830078125, + "reward_std": 0.27600592374801636, + "rewards/accuracy_reward/mean": 0.150390625, + "rewards/accuracy_reward/std": 0.35780346393585205, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9326171875, + "rewards/tag_count_reward/std": 0.1898876577615738, + "step": 345 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.142578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2014.0, + "completions/mean_length": 1194.251953125, + "completions/mean_terminated_length": 1052.2847900390625, + "completions/min_length": 222.0, + "completions/min_terminated_length": 222.0, + "epoch": 0.11811897243321669, + "grad_norm": 0.41605186462402344, + "kl": 0.055908203125, + "learning_rate": 9.991032539983166e-07, + "loss": 0.1502, + "num_tokens": 234637725.0, + "reward": 1.0078125, + "reward_std": 0.2899217903614044, + "rewards/accuracy_reward/mean": 0.119140625, + "rewards/accuracy_reward/std": 0.32427072525024414, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.888671875, + "rewards/tag_count_reward/std": 0.2419114112854004, + "step": 346 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.115234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 1016.5390625, + "completions/mean_terminated_length": 882.1986694335938, + "completions/min_length": 113.0, + "completions/min_terminated_length": 113.0, + "epoch": 0.11846035674660749, + "grad_norm": 0.5375961065292358, + "kl": 0.05950927734375, + "learning_rate": 9.990691070681169e-07, + "loss": 0.1481, + "num_tokens": 235232241.0, + "reward": 1.08837890625, + "reward_std": 0.27179259061813354, + "rewards/accuracy_reward/mean": 0.173828125, + "rewards/accuracy_reward/std": 0.3793322443962097, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.91455078125, + "rewards/tag_count_reward/std": 0.2170523852109909, + "step": 347 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14453125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 1119.318359375, + "completions/mean_terminated_length": 962.4177856445312, + "completions/min_length": 221.0, + "completions/min_terminated_length": 221.0, + "epoch": 0.11880174105999829, + "grad_norm": 0.6744995713233948, + "kl": 0.070068359375, + "learning_rate": 9.990343227664552e-07, + "loss": 0.1869, + "num_tokens": 235882772.0, + "reward": 1.04736328125, + "reward_std": 0.35462021827697754, + "rewards/accuracy_reward/mean": 0.1640625, + "rewards/accuracy_reward/std": 0.37069445848464966, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.88330078125, + "rewards/tag_count_reward/std": 0.25253984332084656, + "step": 348 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 1082.482421875, + "completions/mean_terminated_length": 963.9100952148438, + "completions/min_length": 231.0, + "completions/min_terminated_length": 231.0, + "epoch": 0.11914312537338909, + "grad_norm": 0.274726927280426, + "kl": 0.0433349609375, + "learning_rate": 9.98998901142702e-07, + "loss": 0.0858, + "num_tokens": 236514219.0, + "reward": 1.05810546875, + "reward_std": 0.2895505130290985, + "rewards/accuracy_reward/mean": 0.142578125, + "rewards/accuracy_reward/std": 0.3499840497970581, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.91552734375, + "rewards/tag_count_reward/std": 0.20348748564720154, + "step": 349 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.150390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 1148.9453125, + "completions/mean_terminated_length": 989.8023071289062, + "completions/min_length": 233.0, + "completions/min_terminated_length": 233.0, + "epoch": 0.11948450968677989, + "grad_norm": 0.8284821510314941, + "kl": 0.0533447265625, + "learning_rate": 9.989628422471316e-07, + "loss": 0.1744, + "num_tokens": 237173295.0, + "reward": 0.93017578125, + "reward_std": 0.30755260586738586, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.87744140625, + "rewards/tag_count_reward/std": 0.247776597738266, + "step": 350 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.134765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 1150.068359375, + "completions/mean_terminated_length": 1010.2099609375, + "completions/min_length": 281.0, + "completions/min_terminated_length": 281.0, + "epoch": 0.1198258940001707, + "grad_norm": 343831.53125, + "kl": 2177.0279541015625, + "learning_rate": 9.989261461309232e-07, + "loss": 87.8897, + "num_tokens": 237841346.0, + "reward": 1.0849609375, + "reward_std": 0.2732813358306885, + "rewards/accuracy_reward/mean": 0.1875, + "rewards/accuracy_reward/std": 0.39070644974708557, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9033203125, + "rewards/tag_count_reward/std": 0.22377894818782806, + "step": 351 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 1095.2578125, + "completions/mean_terminated_length": 944.37109375, + "completions/min_length": 109.0, + "completions/min_terminated_length": 109.0, + "epoch": 0.1201672783135615, + "grad_norm": 2.281174898147583, + "kl": 0.10009765625, + "learning_rate": 9.9888881284616e-07, + "loss": 0.1497, + "num_tokens": 238469238.0, + "reward": 0.9580078125, + "reward_std": 0.27261272072792053, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9013671875, + "rewards/tag_count_reward/std": 0.224017933011055, + "step": 352 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 1179.5546875, + "completions/mean_terminated_length": 1023.4746704101562, + "completions/min_length": 178.0, + "completions/min_terminated_length": 178.0, + "epoch": 0.1205086626269523, + "grad_norm": 117.5628433227539, + "kl": 1.05743408203125, + "learning_rate": 9.9885084244583e-07, + "loss": 0.213, + "num_tokens": 239156018.0, + "reward": 0.9990234375, + "reward_std": 0.29136621952056885, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8935546875, + "rewards/tag_count_reward/std": 0.2328195720911026, + "step": 353 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.189453125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 1276.390625, + "completions/mean_terminated_length": 1096.03857421875, + "completions/min_length": 201.0, + "completions/min_terminated_length": 201.0, + "epoch": 0.1208500469403431, + "grad_norm": 3.7248942852020264, + "kl": 0.09783935546875, + "learning_rate": 9.988122349838247e-07, + "loss": 0.1338, + "num_tokens": 239891034.0, + "reward": 0.9345703125, + "reward_std": 0.2773207426071167, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8525390625, + "rewards/tag_count_reward/std": 0.2726678252220154, + "step": 354 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.162109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2015.0, + "completions/mean_length": 1155.509765625, + "completions/mean_terminated_length": 982.8368530273438, + "completions/min_length": 279.0, + "completions/min_terminated_length": 279.0, + "epoch": 0.1211914312537339, + "grad_norm": 0.44104957580566406, + "kl": 0.0574951171875, + "learning_rate": 9.987729905149411e-07, + "loss": 0.1821, + "num_tokens": 240557519.0, + "reward": 1.03271484375, + "reward_std": 0.33100438117980957, + "rewards/accuracy_reward/mean": 0.15927419066429138, + "rewards/accuracy_reward/std": 0.3663010001182556, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.87841796875, + "rewards/tag_count_reward/std": 0.25168323516845703, + "step": 355 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08984375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2017.0, + "completions/mean_length": 1106.876953125, + "completions/mean_terminated_length": 1013.9763793945312, + "completions/min_length": 223.0, + "completions/min_terminated_length": 223.0, + "epoch": 0.1215328155671247, + "grad_norm": 0.24413111805915833, + "kl": 0.0404052734375, + "learning_rate": 9.98733109094879e-07, + "loss": 0.1194, + "num_tokens": 241204832.0, + "reward": 1.041015625, + "reward_std": 0.22742190957069397, + "rewards/accuracy_reward/mean": 0.109375, + "rewards/accuracy_reward/std": 0.31241437792778015, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.931640625, + "rewards/tag_count_reward/std": 0.1836380660533905, + "step": 356 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.16796875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 1181.310546875, + "completions/mean_terminated_length": 1006.3450927734375, + "completions/min_length": 286.0, + "completions/min_terminated_length": 286.0, + "epoch": 0.1218741998805155, + "grad_norm": 0.2896995544433594, + "kl": 0.04241943359375, + "learning_rate": 9.98692590780243e-07, + "loss": 0.1404, + "num_tokens": 241885727.0, + "reward": 1.01611328125, + "reward_std": 0.32733917236328125, + "rewards/accuracy_reward/mean": 0.14453125, + "rewards/accuracy_reward/std": 0.35197147727012634, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.87158203125, + "rewards/tag_count_reward/std": 0.2564968764781952, + "step": 357 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12890625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 1061.03125, + "completions/mean_terminated_length": 914.9776000976562, + "completions/min_length": 182.0, + "completions/min_terminated_length": 182.0, + "epoch": 0.1222155841939063, + "grad_norm": 0.2189946472644806, + "kl": 0.04302978515625, + "learning_rate": 9.986514356285412e-07, + "loss": 0.129, + "num_tokens": 242519743.0, + "reward": 1.095703125, + "reward_std": 0.274413526058197, + "rewards/accuracy_reward/mean": 0.177734375, + "rewards/accuracy_reward/std": 0.3826628625392914, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.91796875, + "rewards/tag_count_reward/std": 0.19562077522277832, + "step": 358 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.169921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 1221.43359375, + "completions/mean_terminated_length": 1052.2305908203125, + "completions/min_length": 275.0, + "completions/min_terminated_length": 275.0, + "epoch": 0.1225569685072971, + "grad_norm": 0.20168673992156982, + "kl": 0.04083251953125, + "learning_rate": 9.986096436981862e-07, + "loss": 0.1191, + "num_tokens": 243226989.0, + "reward": 0.99755859375, + "reward_std": 0.2901397943496704, + "rewards/accuracy_reward/mean": 0.109375, + "rewards/accuracy_reward/std": 0.31241437792778015, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.88818359375, + "rewards/tag_count_reward/std": 0.23107852041721344, + "step": 359 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.142578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 1172.826171875, + "completions/mean_terminated_length": 1027.296142578125, + "completions/min_length": 247.0, + "completions/min_terminated_length": 247.0, + "epoch": 0.1228983528206879, + "grad_norm": 0.4305398166179657, + "kl": 0.046630859375, + "learning_rate": 9.985672150484937e-07, + "loss": 0.1307, + "num_tokens": 243910340.0, + "reward": 0.9375, + "reward_std": 0.27414533495903015, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.88671875, + "rewards/tag_count_reward/std": 0.23998405039310455, + "step": 360 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2007.0, + "completions/mean_length": 1167.765625, + "completions/mean_terminated_length": 995.0093383789062, + "completions/min_length": 164.0, + "completions/min_terminated_length": 164.0, + "epoch": 0.1232397371340787, + "grad_norm": 0.24642539024353027, + "kl": 0.04925537109375, + "learning_rate": 9.985241497396835e-07, + "loss": 0.1211, + "num_tokens": 244590060.0, + "reward": 1.04150390625, + "reward_std": 0.3077473044395447, + "rewards/accuracy_reward/mean": 0.14453125, + "rewards/accuracy_reward/std": 0.35197147727012634, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.89697265625, + "rewards/tag_count_reward/std": 0.2266613245010376, + "step": 361 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 1058.2890625, + "completions/mean_terminated_length": 946.4086303710938, + "completions/min_length": 229.0, + "completions/min_terminated_length": 229.0, + "epoch": 0.1235811214474695, + "grad_norm": 0.2917609214782715, + "kl": 0.0450439453125, + "learning_rate": 9.984804478328792e-07, + "loss": 0.1059, + "num_tokens": 245214256.0, + "reward": 1.029296875, + "reward_std": 0.23584429919719696, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.923828125, + "rewards/tag_count_reward/std": 0.2070399820804596, + "step": 362 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.134765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2024.0, + "completions/mean_length": 1088.248046875, + "completions/mean_terminated_length": 938.7607421875, + "completions/min_length": 209.0, + "completions/min_terminated_length": 209.0, + "epoch": 0.1239225057608603, + "grad_norm": 0.21121852099895477, + "kl": 0.0457763671875, + "learning_rate": 9.98436109390107e-07, + "loss": 0.127, + "num_tokens": 245853839.0, + "reward": 1.068359375, + "reward_std": 0.3218876123428345, + "rewards/accuracy_reward/mean": 0.173828125, + "rewards/accuracy_reward/std": 0.3793322443962097, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.89453125, + "rewards/tag_count_reward/std": 0.23168610036373138, + "step": 363 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.224609375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 1301.646484375, + "completions/mean_terminated_length": 1085.4483642578125, + "completions/min_length": 289.0, + "completions/min_terminated_length": 289.0, + "epoch": 0.1242638900742511, + "grad_norm": 0.4051723778247833, + "kl": 0.0487060546875, + "learning_rate": 9.983911344742979e-07, + "loss": 0.1438, + "num_tokens": 246587770.0, + "reward": 0.97314453125, + "reward_std": 0.3036273121833801, + "rewards/accuracy_reward/mean": 0.134765625, + "rewards/accuracy_reward/std": 0.3418070077896118, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.83837890625, + "rewards/tag_count_reward/std": 0.2800033390522003, + "step": 364 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.140625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 1169.8203125, + "completions/mean_terminated_length": 1026.1181640625, + "completions/min_length": 269.0, + "completions/min_terminated_length": 269.0, + "epoch": 0.12460527438764189, + "grad_norm": 1.240547776222229, + "kl": 0.070068359375, + "learning_rate": 9.983455231492852e-07, + "loss": 0.1581, + "num_tokens": 247262510.0, + "reward": 1.03173828125, + "reward_std": 0.31879186630249023, + "rewards/accuracy_reward/mean": 0.140625, + "rewards/accuracy_reward/std": 0.3479743003845215, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.89111328125, + "rewards/tag_count_reward/std": 0.2392178475856781, + "step": 365 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 1166.392578125, + "completions/mean_terminated_length": 1003.1319580078125, + "completions/min_length": 149.0, + "completions/min_terminated_length": 149.0, + "epoch": 0.12494665870103269, + "grad_norm": 0.8436083793640137, + "kl": 0.073974609375, + "learning_rate": 9.982992754798057e-07, + "loss": 0.1818, + "num_tokens": 247936503.0, + "reward": 1.0615234375, + "reward_std": 0.37862253189086914, + "rewards/accuracy_reward/mean": 0.18359375, + "rewards/accuracy_reward/std": 0.3875311613082886, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8779296875, + "rewards/tag_count_reward/std": 0.24924781918525696, + "step": 366 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.173828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 1193.67578125, + "completions/mean_terminated_length": 1013.9243774414062, + "completions/min_length": 231.0, + "completions/min_terminated_length": 231.0, + "epoch": 0.12528804301442348, + "grad_norm": 378.8311462402344, + "kl": 3.0299072265625, + "learning_rate": 9.982523915315e-07, + "loss": 0.2734, + "num_tokens": 248625025.0, + "reward": 0.99462890625, + "reward_std": 0.28291645646095276, + "rewards/accuracy_reward/mean": 0.111328125, + "rewards/accuracy_reward/std": 0.31484565138816833, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.88330078125, + "rewards/tag_count_reward/std": 0.24516746401786804, + "step": 367 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.150390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 1200.595703125, + "completions/mean_terminated_length": 1050.595458984375, + "completions/min_length": 257.0, + "completions/min_terminated_length": 257.0, + "epoch": 0.1256294273278143, + "grad_norm": 4.661107540130615, + "kl": 0.2315673828125, + "learning_rate": 9.982048713709109e-07, + "loss": 0.1408, + "num_tokens": 249312050.0, + "reward": 1.10302734375, + "reward_std": 0.33130943775177, + "rewards/accuracy_reward/mean": 0.208984375, + "rewards/accuracy_reward/std": 0.40698084235191345, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.89404296875, + "rewards/tag_count_reward/std": 0.23066876828670502, + "step": 368 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14453125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 1161.1875, + "completions/mean_terminated_length": 1011.3607177734375, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "epoch": 0.12597081164120508, + "grad_norm": 42.994876861572266, + "kl": 1.1650390625, + "learning_rate": 9.981567150654848e-07, + "loss": 0.1515, + "num_tokens": 249979122.0, + "reward": 1.07666015625, + "reward_std": 0.32884156703948975, + "rewards/accuracy_reward/mean": 0.169921875, + "rewards/accuracy_reward/std": 0.3759314715862274, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.90673828125, + "rewards/tag_count_reward/std": 0.2254990190267563, + "step": 369 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.169921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 1240.078125, + "completions/mean_terminated_length": 1074.6917724609375, + "completions/min_length": 109.0, + "completions/min_terminated_length": 109.0, + "epoch": 0.1263121959545959, + "grad_norm": 2410563.0, + "kl": 15131.19091796875, + "learning_rate": 9.981079226835704e-07, + "loss": 605.8589, + "num_tokens": 250689994.0, + "reward": 1.02685546875, + "reward_std": 0.35881370306015015, + "rewards/accuracy_reward/mean": 0.150390625, + "rewards/accuracy_reward/std": 0.35780346393585205, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.87646484375, + "rewards/tag_count_reward/std": 0.2487695813179016, + "step": 370 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.115234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2020.0, + "completions/mean_length": 1174.806640625, + "completions/mean_terminated_length": 1061.0794677734375, + "completions/min_length": 224.0, + "completions/min_terminated_length": 224.0, + "epoch": 0.12665358026798668, + "grad_norm": 87.1528091430664, + "kl": 0.883056640625, + "learning_rate": 9.9805849429442e-07, + "loss": 0.1969, + "num_tokens": 251368151.0, + "reward": 1.0009765625, + "reward_std": 0.2787174582481384, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9072265625, + "rewards/tag_count_reward/std": 0.22324882447719574, + "step": 371 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.138671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2024.0, + "completions/mean_length": 1187.529296875, + "completions/mean_terminated_length": 1048.9954833984375, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.1269949645813775, + "grad_norm": 0.4278341233730316, + "kl": 0.08721923828125, + "learning_rate": 9.980084299681879e-07, + "loss": 0.1303, + "num_tokens": 252067190.0, + "reward": 0.96435546875, + "reward_std": 0.2680964469909668, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.89208984375, + "rewards/tag_count_reward/std": 0.23240621387958527, + "step": 372 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 1125.94140625, + "completions/mean_terminated_length": 998.9022216796875, + "completions/min_length": 229.0, + "completions/min_terminated_length": 229.0, + "epoch": 0.12733634889476828, + "grad_norm": 0.4684431254863739, + "kl": 0.0633544921875, + "learning_rate": 9.979577297759312e-07, + "loss": 0.1117, + "num_tokens": 252718296.0, + "reward": 1.10400390625, + "reward_std": 0.316215455532074, + "rewards/accuracy_reward/mean": 0.19921875, + "rewards/accuracy_reward/std": 0.39980348944664, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.90478515625, + "rewards/tag_count_reward/std": 0.21860963106155396, + "step": 373 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.173828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 1212.505859375, + "completions/mean_terminated_length": 1036.71630859375, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "epoch": 0.1276777332081591, + "grad_norm": 0.7251440286636353, + "kl": 0.06719970703125, + "learning_rate": 9.979063937896098e-07, + "loss": 0.1669, + "num_tokens": 253413307.0, + "reward": 1.015625, + "reward_std": 0.32031503319740295, + "rewards/accuracy_reward/mean": 0.134765625, + "rewards/accuracy_reward/std": 0.3418070077896118, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.880859375, + "rewards/tag_count_reward/std": 0.24771922826766968, + "step": 374 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 1116.748046875, + "completions/mean_terminated_length": 993.1305541992188, + "completions/min_length": 121.0, + "completions/min_terminated_length": 121.0, + "epoch": 0.12801911752154987, + "grad_norm": 0.4610971510410309, + "kl": 0.05511474609375, + "learning_rate": 9.978544220820858e-07, + "loss": 0.0899, + "num_tokens": 254067322.0, + "reward": 1.0341796875, + "reward_std": 0.27670881152153015, + "rewards/accuracy_reward/mean": 0.125, + "rewards/accuracy_reward/std": 0.3310528099536896, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9130859375, + "rewards/tag_count_reward/std": 0.20515529811382294, + "step": 375 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.17578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 1296.08203125, + "completions/mean_terminated_length": 1135.720458984375, + "completions/min_length": 156.0, + "completions/min_terminated_length": 156.0, + "epoch": 0.1283605018349407, + "grad_norm": 0.5585299730300903, + "kl": 0.0692138671875, + "learning_rate": 9.978018147271232e-07, + "loss": 0.1125, + "num_tokens": 254811396.0, + "reward": 0.9990234375, + "reward_std": 0.2815853953361511, + "rewards/accuracy_reward/mean": 0.107421875, + "rewards/accuracy_reward/std": 0.30995169281959534, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8916015625, + "rewards/tag_count_reward/std": 0.22872866690158844, + "step": 376 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.142578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2022.0, + "completions/mean_length": 1152.908203125, + "completions/mean_terminated_length": 1004.0661010742188, + "completions/min_length": 245.0, + "completions/min_terminated_length": 245.0, + "epoch": 0.12870188614833147, + "grad_norm": 0.8666077256202698, + "kl": 0.1131591796875, + "learning_rate": 9.977485717993885e-07, + "loss": 0.1211, + "num_tokens": 255469333.0, + "reward": 1.01318359375, + "reward_std": 0.22732698917388916, + "rewards/accuracy_reward/mean": 0.11895161122083664, + "rewards/accuracy_reward/std": 0.3240584135055542, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.89794921875, + "rewards/tag_count_reward/std": 0.22548207640647888, + "step": 377 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 1194.314453125, + "completions/mean_terminated_length": 1026.7686767578125, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "epoch": 0.1290432704617223, + "grad_norm": 3.573004961013794, + "kl": 0.22021484375, + "learning_rate": 9.976946933744505e-07, + "loss": 0.1275, + "num_tokens": 256150294.0, + "reward": 1.00341796875, + "reward_std": 0.28133586049079895, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.89794921875, + "rewards/tag_count_reward/std": 0.21943418681621552, + "step": 378 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2031.0, + "completions/mean_length": 1269.986328125, + "completions/mean_terminated_length": 1071.669189453125, + "completions/min_length": 226.0, + "completions/min_terminated_length": 226.0, + "epoch": 0.12938465477511307, + "grad_norm": 5.827714443206787, + "kl": 0.137451171875, + "learning_rate": 9.9764017952878e-07, + "loss": 0.1455, + "num_tokens": 256875423.0, + "reward": 0.95947265625, + "reward_std": 0.28445520997047424, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.86572265625, + "rewards/tag_count_reward/std": 0.2500721216201782, + "step": 379 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.166015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2041.0, + "completions/mean_length": 1251.505859375, + "completions/mean_terminated_length": 1092.953125, + "completions/min_length": 145.0, + "completions/min_terminated_length": 145.0, + "epoch": 0.12972603908850389, + "grad_norm": 0.39755210280418396, + "kl": 0.0858154296875, + "learning_rate": 9.97585030339749e-07, + "loss": 0.1089, + "num_tokens": 257591778.0, + "reward": 1.0390625, + "reward_std": 0.2732459306716919, + "rewards/accuracy_reward/mean": 0.13671875, + "rewards/accuracy_reward/std": 0.3438861668109894, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.90234375, + "rewards/tag_count_reward/std": 0.21269488334655762, + "step": 380 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 1232.404296875, + "completions/mean_terminated_length": 1034.4442138671875, + "completions/min_length": 152.0, + "completions/min_terminated_length": 152.0, + "epoch": 0.13006742340189467, + "grad_norm": 0.8170649409294128, + "kl": 0.09619140625, + "learning_rate": 9.97529245885632e-07, + "loss": 0.1599, + "num_tokens": 258300513.0, + "reward": 0.93017578125, + "reward_std": 0.27194511890411377, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.85595703125, + "rewards/tag_count_reward/std": 0.2642766237258911, + "step": 381 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 1328.248046875, + "completions/mean_terminated_length": 1126.717529296875, + "completions/min_length": 261.0, + "completions/min_terminated_length": 261.0, + "epoch": 0.13040880771528549, + "grad_norm": 2.283935785293579, + "kl": 0.16015625, + "learning_rate": 9.974728262456043e-07, + "loss": 0.1189, + "num_tokens": 259054352.0, + "reward": 0.9736328125, + "reward_std": 0.27440378069877625, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8798828125, + "rewards/tag_count_reward/std": 0.23087425529956818, + "step": 382 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.19921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2028.0, + "completions/mean_length": 1287.685546875, + "completions/mean_terminated_length": 1098.5341796875, + "completions/min_length": 282.0, + "completions/min_terminated_length": 282.0, + "epoch": 0.13075019202867627, + "grad_norm": 0.4480745792388916, + "kl": 0.095703125, + "learning_rate": 9.974157714997438e-07, + "loss": 0.1528, + "num_tokens": 259792623.0, + "reward": 1.0263671875, + "reward_std": 0.3242665231227875, + "rewards/accuracy_reward/mean": 0.1484375, + "rewards/accuracy_reward/std": 0.35588082671165466, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8779296875, + "rewards/tag_count_reward/std": 0.23562633991241455, + "step": 383 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 1343.193359375, + "completions/mean_terminated_length": 1145.8475341796875, + "completions/min_length": 233.0, + "completions/min_terminated_length": 233.0, + "epoch": 0.13109157634206708, + "grad_norm": 0.9114007353782654, + "kl": 0.111328125, + "learning_rate": 9.973580817290295e-07, + "loss": 0.1672, + "num_tokens": 260552306.0, + "reward": 0.958984375, + "reward_std": 0.29250961542129517, + "rewards/accuracy_reward/mean": 0.087890625, + "rewards/accuracy_reward/std": 0.2834126651287079, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.87109375, + "rewards/tag_count_reward/std": 0.24775780737400055, + "step": 384 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.181640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 1200.03515625, + "completions/mean_terminated_length": 1011.8234252929688, + "completions/min_length": 178.0, + "completions/min_terminated_length": 178.0, + "epoch": 0.13143296065545787, + "grad_norm": 1.0169259309768677, + "kl": 0.092041015625, + "learning_rate": 9.972997570153406e-07, + "loss": 0.1167, + "num_tokens": 261251252.0, + "reward": 0.95068359375, + "reward_std": 0.26185333728790283, + "rewards/accuracy_reward/mean": 0.07258064299821854, + "rewards/accuracy_reward/std": 0.25970885157585144, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.88037109375, + "rewards/tag_count_reward/std": 0.238677978515625, + "step": 385 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.205078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 1313.078125, + "completions/mean_terminated_length": 1123.4791259765625, + "completions/min_length": 221.0, + "completions/min_terminated_length": 221.0, + "epoch": 0.13177434496884868, + "grad_norm": 0.7724472284317017, + "kl": 0.095458984375, + "learning_rate": 9.972407974414592e-07, + "loss": 0.1242, + "num_tokens": 261994236.0, + "reward": 1.001953125, + "reward_std": 0.3082892596721649, + "rewards/accuracy_reward/mean": 0.123046875, + "rewards/accuracy_reward/std": 0.32881227135658264, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.87890625, + "rewards/tag_count_reward/std": 0.23509246110916138, + "step": 386 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 1262.189453125, + "completions/mean_terminated_length": 1099.0966796875, + "completions/min_length": 194.0, + "completions/min_terminated_length": 194.0, + "epoch": 0.13211572928223947, + "grad_norm": 0.38649216294288635, + "kl": 0.1070556640625, + "learning_rate": 9.971812030910671e-07, + "loss": 0.1158, + "num_tokens": 262722397.0, + "reward": 0.90771484375, + "reward_std": 0.21668007969856262, + "rewards/accuracy_reward/mean": 0.021484375, + "rewards/accuracy_reward/std": 0.14513419568538666, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.88623046875, + "rewards/tag_count_reward/std": 0.23589536547660828, + "step": 387 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 1263.587890625, + "completions/mean_terminated_length": 1122.610595703125, + "completions/min_length": 257.0, + "completions/min_terminated_length": 257.0, + "epoch": 0.13245711359563028, + "grad_norm": 66.00189208984375, + "kl": 0.6612548828125, + "learning_rate": 9.971209740487478e-07, + "loss": 0.1217, + "num_tokens": 263458058.0, + "reward": 0.98486328125, + "reward_std": 0.25900518894195557, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.89501953125, + "rewards/tag_count_reward/std": 0.23111572861671448, + "step": 388 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.166015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 1226.498046875, + "completions/mean_terminated_length": 1062.9671630859375, + "completions/min_length": 299.0, + "completions/min_terminated_length": 299.0, + "epoch": 0.13279849790902107, + "grad_norm": 0.5843941569328308, + "kl": 0.1075439453125, + "learning_rate": 9.970601103999854e-07, + "loss": 0.1177, + "num_tokens": 264166121.0, + "reward": 0.9853515625, + "reward_std": 0.22699961066246033, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9013671875, + "rewards/tag_count_reward/std": 0.21339233219623566, + "step": 389 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.150390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 1167.7578125, + "completions/mean_terminated_length": 1011.94482421875, + "completions/min_length": 152.0, + "completions/min_terminated_length": 152.0, + "epoch": 0.13313988222241188, + "grad_norm": 1.1302576065063477, + "kl": 0.103515625, + "learning_rate": 9.969986122311648e-07, + "loss": 0.1136, + "num_tokens": 264841197.0, + "reward": 1.03173828125, + "reward_std": 0.2919134795665741, + "rewards/accuracy_reward/mean": 0.140625, + "rewards/accuracy_reward/std": 0.3479743003845215, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.89111328125, + "rewards/tag_count_reward/std": 0.22280485928058624, + "step": 390 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.095703125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 1157.86328125, + "completions/mean_terminated_length": 1063.65869140625, + "completions/min_length": 202.0, + "completions/min_terminated_length": 202.0, + "epoch": 0.13348126653580267, + "grad_norm": 3.2016637325286865, + "kl": 0.1668701171875, + "learning_rate": 9.969364796295712e-07, + "loss": 0.1033, + "num_tokens": 265512919.0, + "reward": 1.111328125, + "reward_std": 0.3207700550556183, + "rewards/accuracy_reward/mean": 0.17578125, + "rewards/accuracy_reward/std": 0.3810062110424042, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.935546875, + "rewards/tag_count_reward/std": 0.17060412466526031, + "step": 391 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.126953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 1230.892578125, + "completions/mean_terminated_length": 1112.0738525390625, + "completions/min_length": 217.0, + "completions/min_terminated_length": 217.0, + "epoch": 0.13382265084919348, + "grad_norm": 2.2589118480682373, + "kl": 0.163330078125, + "learning_rate": 9.968737126833905e-07, + "loss": 0.1171, + "num_tokens": 266216512.0, + "reward": 1.00244140625, + "reward_std": 0.250763475894928, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.91650390625, + "rewards/tag_count_reward/std": 0.19151799380779266, + "step": 392 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.126953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 1169.677734375, + "completions/mean_terminated_length": 1041.95751953125, + "completions/min_length": 286.0, + "completions/min_terminated_length": 286.0, + "epoch": 0.13416403516258427, + "grad_norm": 0.7035832405090332, + "kl": 0.1094970703125, + "learning_rate": 9.96810311481709e-07, + "loss": 0.1273, + "num_tokens": 266890811.0, + "reward": 1.08056640625, + "reward_std": 0.2758799195289612, + "rewards/accuracy_reward/mean": 0.162109375, + "rewards/accuracy_reward/std": 0.3689115643501282, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.91845703125, + "rewards/tag_count_reward/std": 0.19799919426441193, + "step": 393 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 1150.82421875, + "completions/mean_terminated_length": 1027.21337890625, + "completions/min_length": 228.0, + "completions/min_terminated_length": 228.0, + "epoch": 0.13450541947597508, + "grad_norm": 3.396650552749634, + "kl": 0.1527099609375, + "learning_rate": 9.967462761145132e-07, + "loss": 0.1233, + "num_tokens": 267549761.0, + "reward": 0.966796875, + "reward_std": 0.21518467366695404, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.921875, + "rewards/tag_count_reward/std": 0.1921909898519516, + "step": 394 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.103515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 1150.5234375, + "completions/mean_terminated_length": 1046.8931884765625, + "completions/min_length": 331.0, + "completions/min_terminated_length": 331.0, + "epoch": 0.13484680378936587, + "grad_norm": 0.3927019238471985, + "kl": 0.091552734375, + "learning_rate": 9.966816066726895e-07, + "loss": 0.0801, + "num_tokens": 268208397.0, + "reward": 1.154296875, + "reward_std": 0.3116268515586853, + "rewards/accuracy_reward/mean": 0.224609375, + "rewards/accuracy_reward/std": 0.41773295402526855, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9296875, + "rewards/tag_count_reward/std": 0.1835651993751526, + "step": 395 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.142578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 1194.126953125, + "completions/mean_terminated_length": 1052.1390380859375, + "completions/min_length": 177.0, + "completions/min_terminated_length": 177.0, + "epoch": 0.13518818810275668, + "grad_norm": 0.8550708293914795, + "kl": 0.1165771484375, + "learning_rate": 9.96616303248024e-07, + "loss": 0.1276, + "num_tokens": 268900174.0, + "reward": 1.0263671875, + "reward_std": 0.2901027798652649, + "rewards/accuracy_reward/mean": 0.111328125, + "rewards/accuracy_reward/std": 0.31484565138816833, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9150390625, + "rewards/tag_count_reward/std": 0.1949920952320099, + "step": 396 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.146484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 1204.087890625, + "completions/mean_terminated_length": 1059.251708984375, + "completions/min_length": 214.0, + "completions/min_terminated_length": 214.0, + "epoch": 0.13552957241614746, + "grad_norm": 0.7503846287727356, + "kl": 0.127197265625, + "learning_rate": 9.965503659332037e-07, + "loss": 0.1111, + "num_tokens": 269587771.0, + "reward": 1.03857421875, + "reward_std": 0.31176069378852844, + "rewards/accuracy_reward/mean": 0.130859375, + "rewards/accuracy_reward/std": 0.33757632970809937, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.90771484375, + "rewards/tag_count_reward/std": 0.2048913985490799, + "step": 397 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08984375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 1140.435546875, + "completions/mean_terminated_length": 1050.84765625, + "completions/min_length": 167.0, + "completions/min_terminated_length": 167.0, + "epoch": 0.13587095672953828, + "grad_norm": 0.6859690546989441, + "kl": 0.0980224609375, + "learning_rate": 9.964837948218138e-07, + "loss": 0.1032, + "num_tokens": 270245450.0, + "reward": 1.080078125, + "reward_std": 0.21009346842765808, + "rewards/accuracy_reward/mean": 0.14453125, + "rewards/accuracy_reward/std": 0.35197147727012634, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.935546875, + "rewards/tag_count_reward/std": 0.16550962626934052, + "step": 398 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.126953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 1248.32421875, + "completions/mean_terminated_length": 1132.040283203125, + "completions/min_length": 199.0, + "completions/min_terminated_length": 199.0, + "epoch": 0.13621234104292906, + "grad_norm": 2.203038215637207, + "kl": 0.1400146484375, + "learning_rate": 9.964165900083402e-07, + "loss": 0.1197, + "num_tokens": 270966608.0, + "reward": 1.05810546875, + "reward_std": 0.3260589838027954, + "rewards/accuracy_reward/mean": 0.142578125, + "rewards/accuracy_reward/std": 0.3499840497970581, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.91552734375, + "rewards/tag_count_reward/std": 0.19614213705062866, + "step": 399 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 1145.833984375, + "completions/mean_terminated_length": 1021.5355834960938, + "completions/min_length": 129.0, + "completions/min_terminated_length": 129.0, + "epoch": 0.13655372535631988, + "grad_norm": 7.622913837432861, + "kl": 0.250732421875, + "learning_rate": 9.963487515881678e-07, + "loss": 0.1629, + "num_tokens": 271637531.0, + "reward": 1.05419921875, + "reward_std": 0.2815008759498596, + "rewards/accuracy_reward/mean": 0.140625, + "rewards/accuracy_reward/std": 0.3479743003845215, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.91357421875, + "rewards/tag_count_reward/std": 0.2032666653394699, + "step": 400 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.115234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2007.0, + "completions/mean_length": 1098.95703125, + "completions/mean_terminated_length": 975.3510131835938, + "completions/min_length": 164.0, + "completions/min_terminated_length": 164.0, + "epoch": 0.1368951096697107, + "grad_norm": 0.43843021988868713, + "kl": 0.1151123046875, + "learning_rate": 9.962802796575811e-07, + "loss": 0.1067, + "num_tokens": 272284245.0, + "reward": 1.11767578125, + "reward_std": 0.24404722452163696, + "rewards/accuracy_reward/mean": 0.185546875, + "rewards/accuracy_reward/std": 0.38912075757980347, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.93212890625, + "rewards/tag_count_reward/std": 0.17669491469860077, + "step": 401 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.140625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 1223.072265625, + "completions/mean_terminated_length": 1088.0841064453125, + "completions/min_length": 301.0, + "completions/min_terminated_length": 301.0, + "epoch": 0.13723649398310148, + "grad_norm": 0.9861530065536499, + "kl": 0.133056640625, + "learning_rate": 9.96211174313763e-07, + "loss": 0.1246, + "num_tokens": 272975242.0, + "reward": 1.06884765625, + "reward_std": 0.28927093744277954, + "rewards/accuracy_reward/mean": 0.154296875, + "rewards/accuracy_reward/std": 0.36158639192581177, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.91455078125, + "rewards/tag_count_reward/std": 0.19571784138679504, + "step": 402 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 1087.255859375, + "completions/mean_terminated_length": 969.2697143554688, + "completions/min_length": 192.0, + "completions/min_terminated_length": 192.0, + "epoch": 0.1375778782964923, + "grad_norm": 0.6500340700149536, + "kl": 0.1126708984375, + "learning_rate": 9.961414356547962e-07, + "loss": 0.0997, + "num_tokens": 273616125.0, + "reward": 1.09423828125, + "reward_std": 0.31288596987724304, + "rewards/accuracy_reward/mean": 0.171875, + "rewards/accuracy_reward/std": 0.3776407241821289, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.92236328125, + "rewards/tag_count_reward/std": 0.1855938732624054, + "step": 403 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 1189.474609375, + "completions/mean_terminated_length": 1075.5111083984375, + "completions/min_length": 232.0, + "completions/min_terminated_length": 232.0, + "epoch": 0.13791926260988308, + "grad_norm": 0.7278398871421814, + "kl": 0.099853515625, + "learning_rate": 9.960710637796617e-07, + "loss": 0.0855, + "num_tokens": 274301440.0, + "reward": 1.0654296875, + "reward_std": 0.28713470697402954, + "rewards/accuracy_reward/mean": 0.146484375, + "rewards/accuracy_reward/std": 0.35393697023391724, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9189453125, + "rewards/tag_count_reward/std": 0.1966511756181717, + "step": 404 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.087890625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2029.0, + "completions/mean_length": 1110.943359375, + "completions/mean_terminated_length": 1020.6488037109375, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.1382606469232739, + "grad_norm": 0.843716561794281, + "kl": 0.09375, + "learning_rate": 9.960000587882396e-07, + "loss": 0.0897, + "num_tokens": 274946707.0, + "reward": 1.0576171875, + "reward_std": 0.21474316716194153, + "rewards/accuracy_reward/mean": 0.115234375, + "rewards/accuracy_reward/std": 0.3196168541908264, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9423828125, + "rewards/tag_count_reward/std": 0.16507895290851593, + "step": 405 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 1081.494140625, + "completions/mean_terminated_length": 981.5107421875, + "completions/min_length": 243.0, + "completions/min_terminated_length": 243.0, + "epoch": 0.13860203123666467, + "grad_norm": 0.6134137511253357, + "kl": 0.1099853515625, + "learning_rate": 9.95928420781309e-07, + "loss": 0.0624, + "num_tokens": 275574352.0, + "reward": 1.1337890625, + "reward_std": 0.21922734379768372, + "rewards/accuracy_reward/mean": 0.18359375, + "rewards/accuracy_reward/std": 0.3875311613082886, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9501953125, + "rewards/tag_count_reward/std": 0.1406233012676239, + "step": 406 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2021.0, + "completions/mean_length": 1095.412109375, + "completions/mean_terminated_length": 968.96240234375, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.1389434155500555, + "grad_norm": 12.477898597717285, + "kl": 0.269775390625, + "learning_rate": 9.95856149860546e-07, + "loss": 0.1398, + "num_tokens": 276222131.0, + "reward": 1.05126953125, + "reward_std": 0.2644086480140686, + "rewards/accuracy_reward/mean": 0.13306452333927155, + "rewards/accuracy_reward/std": 0.3399873673915863, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.92236328125, + "rewards/tag_count_reward/std": 0.19460150599479675, + "step": 407 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.123046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 1148.38671875, + "completions/mean_terminated_length": 1022.160400390625, + "completions/min_length": 28.0, + "completions/min_terminated_length": 28.0, + "epoch": 0.13928479986344627, + "grad_norm": 2.7371554374694824, + "kl": 0.194091796875, + "learning_rate": 9.957832461285267e-07, + "loss": 0.1006, + "num_tokens": 276897081.0, + "reward": 0.99755859375, + "reward_std": 0.250779390335083, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.92138671875, + "rewards/tag_count_reward/std": 0.1818491518497467, + "step": 408 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0703125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 1098.951171875, + "completions/mean_terminated_length": 1027.1744384765625, + "completions/min_length": 266.0, + "completions/min_terminated_length": 266.0, + "epoch": 0.1396261841768371, + "grad_norm": 0.6406298875808716, + "kl": 0.1231689453125, + "learning_rate": 9.957097096887246e-07, + "loss": 0.0937, + "num_tokens": 277536512.0, + "reward": 1.0390625, + "reward_std": 0.2268882691860199, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.94921875, + "rewards/tag_count_reward/std": 0.15118376910686493, + "step": 409 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.087890625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 1129.705078125, + "completions/mean_terminated_length": 1041.2183837890625, + "completions/min_length": 219.0, + "completions/min_terminated_length": 219.0, + "epoch": 0.13996756849022787, + "grad_norm": 1.1640084981918335, + "kl": 0.156494140625, + "learning_rate": 9.95635540645511e-07, + "loss": 0.0856, + "num_tokens": 278205705.0, + "reward": 1.0146484375, + "reward_std": 0.23164458572864532, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9306640625, + "rewards/tag_count_reward/std": 0.1812576949596405, + "step": 410 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 1050.11328125, + "completions/mean_terminated_length": 996.7283935546875, + "completions/min_length": 200.0, + "completions/min_terminated_length": 200.0, + "epoch": 0.1403089528036187, + "grad_norm": 0.9455592036247253, + "kl": 0.128662109375, + "learning_rate": 9.95560739104155e-07, + "loss": 0.0635, + "num_tokens": 278820243.0, + "reward": 1.16796875, + "reward_std": 0.24406027793884277, + "rewards/accuracy_reward/mean": 0.205078125, + "rewards/accuracy_reward/std": 0.4041535556316376, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.962890625, + "rewards/tag_count_reward/std": 0.12740769982337952, + "step": 411 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.103515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2013.0, + "completions/mean_length": 1094.158203125, + "completions/mean_terminated_length": 984.0195922851562, + "completions/min_length": 156.0, + "completions/min_terminated_length": 156.0, + "epoch": 0.14065033711700947, + "grad_norm": 1.8893964290618896, + "kl": 0.1865234375, + "learning_rate": 9.95485305170824e-07, + "loss": 0.0947, + "num_tokens": 279457684.0, + "reward": 1.0546875, + "reward_std": 0.24043290317058563, + "rewards/accuracy_reward/mean": 0.11895161122083664, + "rewards/accuracy_reward/std": 0.3240584135055542, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.939453125, + "rewards/tag_count_reward/std": 0.16550962626934052, + "step": 412 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07421875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 1008.185546875, + "completions/mean_terminated_length": 924.8248291015625, + "completions/min_length": 144.0, + "completions/min_terminated_length": 144.0, + "epoch": 0.14099172143040029, + "grad_norm": 0.7469418048858643, + "kl": 0.1116943359375, + "learning_rate": 9.95409238952583e-07, + "loss": 0.1202, + "num_tokens": 280042723.0, + "reward": 1.1484375, + "reward_std": 0.2419055700302124, + "rewards/accuracy_reward/mean": 0.203125, + "rewards/accuracy_reward/std": 0.4027182459831238, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9453125, + "rewards/tag_count_reward/std": 0.16007427871227264, + "step": 413 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 1070.05859375, + "completions/mean_terminated_length": 987.1821899414062, + "completions/min_length": 175.0, + "completions/min_terminated_length": 175.0, + "epoch": 0.14133310574379107, + "grad_norm": 0.821952223777771, + "kl": 0.1478271484375, + "learning_rate": 9.953325405573935e-07, + "loss": 0.1546, + "num_tokens": 280678113.0, + "reward": 1.08203125, + "reward_std": 0.24936708807945251, + "rewards/accuracy_reward/mean": 0.130859375, + "rewards/accuracy_reward/std": 0.33757632970809937, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.951171875, + "rewards/tag_count_reward/std": 0.15101934969425201, + "step": 414 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.083984375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 1101.2890625, + "completions/mean_terminated_length": 1014.4904174804688, + "completions/min_length": 238.0, + "completions/min_terminated_length": 238.0, + "epoch": 0.14167449005718188, + "grad_norm": 0.9400856494903564, + "kl": 0.1046142578125, + "learning_rate": 9.952552100941155e-07, + "loss": 0.1123, + "num_tokens": 281315973.0, + "reward": 1.08935546875, + "reward_std": 0.26338285207748413, + "rewards/accuracy_reward/mean": 0.142578125, + "rewards/accuracy_reward/std": 0.3499840497970581, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.94677734375, + "rewards/tag_count_reward/std": 0.1531609743833542, + "step": 415 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.080078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 1065.1875, + "completions/mean_terminated_length": 979.6348876953125, + "completions/min_length": 117.0, + "completions/min_terminated_length": 117.0, + "epoch": 0.14201587437057267, + "grad_norm": 1.530302882194519, + "kl": 0.161865234375, + "learning_rate": 9.951772476725047e-07, + "loss": 0.1158, + "num_tokens": 281939077.0, + "reward": 1.126953125, + "reward_std": 0.24623417854309082, + "rewards/accuracy_reward/mean": 0.19140625, + "rewards/accuracy_reward/std": 0.3937928080558777, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.935546875, + "rewards/tag_count_reward/std": 0.18036192655563354, + "step": 416 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1992.0, + "completions/mean_length": 1130.302734375, + "completions/mean_terminated_length": 1022.1026611328125, + "completions/min_length": 250.0, + "completions/min_terminated_length": 250.0, + "epoch": 0.14235725868396348, + "grad_norm": 0.9781035780906677, + "kl": 0.0975341796875, + "learning_rate": 9.950986534032149e-07, + "loss": 0.0945, + "num_tokens": 282589616.0, + "reward": 1.0986328125, + "reward_std": 0.33007973432540894, + "rewards/accuracy_reward/mean": 0.171875, + "rewards/accuracy_reward/std": 0.3776407241821289, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9267578125, + "rewards/tag_count_reward/std": 0.19347688555717468, + "step": 417 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08984375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 1083.5, + "completions/mean_terminated_length": 988.2918701171875, + "completions/min_length": 165.0, + "completions/min_terminated_length": 165.0, + "epoch": 0.14269864299735427, + "grad_norm": 0.8044054508209229, + "kl": 0.116943359375, + "learning_rate": 9.950194273977964e-07, + "loss": 0.0649, + "num_tokens": 283220176.0, + "reward": 0.990234375, + "reward_std": 0.21570071578025818, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.939453125, + "rewards/tag_count_reward/std": 0.16402500867843628, + "step": 418 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0703125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 1052.771484375, + "completions/mean_terminated_length": 977.5021362304688, + "completions/min_length": 166.0, + "completions/min_terminated_length": 166.0, + "epoch": 0.14304002731074508, + "grad_norm": 0.7905445098876953, + "kl": 0.1112060546875, + "learning_rate": 9.949395697686958e-07, + "loss": 0.0735, + "num_tokens": 283831259.0, + "reward": 1.1220703125, + "reward_std": 0.2493770867586136, + "rewards/accuracy_reward/mean": 0.16796875, + "rewards/accuracy_reward/std": 0.374204158782959, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9541015625, + "rewards/tag_count_reward/std": 0.14366182684898376, + "step": 419 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.087890625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 1091.595703125, + "completions/mean_terminated_length": 999.4368286132812, + "completions/min_length": 194.0, + "completions/min_terminated_length": 194.0, + "epoch": 0.14338141162413587, + "grad_norm": 0.7350541353225708, + "kl": 0.151123046875, + "learning_rate": 9.948590806292565e-07, + "loss": 0.0946, + "num_tokens": 284474604.0, + "reward": 0.97607421875, + "reward_std": 0.1850041151046753, + "rewards/accuracy_reward/mean": 0.04032257944345474, + "rewards/accuracy_reward/std": 0.19691328704357147, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.93701171875, + "rewards/tag_count_reward/std": 0.16347575187683105, + "step": 420 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 1126.125, + "completions/mean_terminated_length": 1017.4323120117188, + "completions/min_length": 157.0, + "completions/min_terminated_length": 157.0, + "epoch": 0.14372279593752668, + "grad_norm": 19.234102249145508, + "kl": 0.6923828125, + "learning_rate": 9.947779600937181e-07, + "loss": 0.1407, + "num_tokens": 285135980.0, + "reward": 1.06689453125, + "reward_std": 0.27593308687210083, + "rewards/accuracy_reward/mean": 0.134765625, + "rewards/accuracy_reward/std": 0.3418070077896118, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.93212890625, + "rewards/tag_count_reward/std": 0.1794423907995224, + "step": 421 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.087890625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2023.0, + "completions/mean_length": 1111.66015625, + "completions/mean_terminated_length": 1021.4346923828125, + "completions/min_length": 96.0, + "completions/min_terminated_length": 96.0, + "epoch": 0.14406418025091747, + "grad_norm": 33.088340759277344, + "kl": 0.493408203125, + "learning_rate": 9.946962082772163e-07, + "loss": 0.1197, + "num_tokens": 285783246.0, + "reward": 1.00830078125, + "reward_std": 0.2411033660173416, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.93017578125, + "rewards/tag_count_reward/std": 0.1800537258386612, + "step": 422 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 1027.78515625, + "completions/mean_terminated_length": 959.7708740234375, + "completions/min_length": 151.0, + "completions/min_terminated_length": 151.0, + "epoch": 0.14440556456430828, + "grad_norm": 9.780620574951172, + "kl": 0.3726806640625, + "learning_rate": 9.946138252957827e-07, + "loss": 0.0718, + "num_tokens": 286387408.0, + "reward": 1.12353515625, + "reward_std": 0.2738574743270874, + "rewards/accuracy_reward/mean": 0.177734375, + "rewards/accuracy_reward/std": 0.3826628625392914, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.94580078125, + "rewards/tag_count_reward/std": 0.15985849499702454, + "step": 423 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07421875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 1135.7109375, + "completions/mean_terminated_length": 1062.57373046875, + "completions/min_length": 234.0, + "completions/min_terminated_length": 234.0, + "epoch": 0.14474694887769907, + "grad_norm": 2.2094523906707764, + "kl": 0.220947265625, + "learning_rate": 9.945308112663455e-07, + "loss": 0.0801, + "num_tokens": 287045020.0, + "reward": 1.0849609375, + "reward_std": 0.2298390418291092, + "rewards/accuracy_reward/mean": 0.138671875, + "rewards/accuracy_reward/std": 0.34594178199768066, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9462890625, + "rewards/tag_count_reward/std": 0.15178616344928741, + "step": 424 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.068359375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2005.0, + "completions/mean_length": 1094.0234375, + "completions/mean_terminated_length": 1024.025146484375, + "completions/min_length": 212.0, + "completions/min_terminated_length": 212.0, + "epoch": 0.14508833319108988, + "grad_norm": 1.2251715660095215, + "kl": 0.12353515625, + "learning_rate": 9.94447166306727e-07, + "loss": 0.1319, + "num_tokens": 287689400.0, + "reward": 1.04296875, + "reward_std": 0.2499944269657135, + "rewards/accuracy_reward/mean": 0.107421875, + "rewards/accuracy_reward/std": 0.30995169281959534, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.935546875, + "rewards/tag_count_reward/std": 0.1734480857849121, + "step": 425 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.095703125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 1104.681640625, + "completions/mean_terminated_length": 1004.8487548828125, + "completions/min_length": 126.0, + "completions/min_terminated_length": 126.0, + "epoch": 0.14542971750448067, + "grad_norm": 0.9261499047279358, + "kl": 0.15966796875, + "learning_rate": 9.94362890535647e-07, + "loss": 0.1016, + "num_tokens": 288330581.0, + "reward": 1.04736328125, + "reward_std": 0.27268338203430176, + "rewards/accuracy_reward/mean": 0.11491935700178146, + "rewards/accuracy_reward/std": 0.3192465901374817, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.93603515625, + "rewards/tag_count_reward/std": 0.16384358704090118, + "step": 426 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.080078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 1078.265625, + "completions/mean_terminated_length": 993.8514404296875, + "completions/min_length": 175.0, + "completions/min_terminated_length": 175.0, + "epoch": 0.14577110181787148, + "grad_norm": 14.89603328704834, + "kl": 0.5328369140625, + "learning_rate": 9.942779840727185e-07, + "loss": 0.1138, + "num_tokens": 288961213.0, + "reward": 1.04638671875, + "reward_std": 0.2174864411354065, + "rewards/accuracy_reward/mean": 0.1015625, + "rewards/accuracy_reward/std": 0.30236753821372986, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.94482421875, + "rewards/tag_count_reward/std": 0.1532669961452484, + "step": 427 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.064453125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 1067.38671875, + "completions/mean_terminated_length": 999.828857421875, + "completions/min_length": 200.0, + "completions/min_terminated_length": 200.0, + "epoch": 0.14611248613126226, + "grad_norm": 2.4918482303619385, + "kl": 0.1580810546875, + "learning_rate": 9.941924470384515e-07, + "loss": 0.064, + "num_tokens": 289578515.0, + "reward": 1.197265625, + "reward_std": 0.26387596130371094, + "rewards/accuracy_reward/mean": 0.240234375, + "rewards/accuracy_reward/std": 0.4276435375213623, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.95703125, + "rewards/tag_count_reward/std": 0.13403046131134033, + "step": 428 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.091796875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2041.0, + "completions/mean_length": 1083.28515625, + "completions/mean_terminated_length": 985.7763671875, + "completions/min_length": 214.0, + "completions/min_terminated_length": 214.0, + "epoch": 0.14645387044465308, + "grad_norm": 0.9718239307403564, + "kl": 0.1097412109375, + "learning_rate": 9.941062795542496e-07, + "loss": 0.0867, + "num_tokens": 290212341.0, + "reward": 1.0380859375, + "reward_std": 0.21170061826705933, + "rewards/accuracy_reward/mean": 0.099609375, + "rewards/accuracy_reward/std": 0.29977133870124817, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.9365234375, + "rewards/tag_count_reward/std": 0.1779806911945343, + "step": 429 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.068359375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2029.0, + "completions/mean_length": 1034.599609375, + "completions/mean_terminated_length": 960.2410278320312, + "completions/min_length": 199.0, + "completions/min_terminated_length": 199.0, + "epoch": 0.14679525475804386, + "grad_norm": 0.8444366455078125, + "kl": 0.1531982421875, + "learning_rate": 9.94019481742412e-07, + "loss": 0.1347, + "num_tokens": 290828648.0, + "reward": 1.11376953125, + "reward_std": 0.2682647109031677, + "rewards/accuracy_reward/mean": 0.162109375, + "rewards/accuracy_reward/std": 0.3689115643501282, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.95166015625, + "rewards/tag_count_reward/std": 0.1523853987455368, + "step": 430 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.068359375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2019.0, + "completions/mean_length": 1074.7421875, + "completions/mean_terminated_length": 1003.3291015625, + "completions/min_length": 229.0, + "completions/min_terminated_length": 229.0, + "epoch": 0.14713663907143468, + "grad_norm": 0.5404824614524841, + "kl": 0.1015625, + "learning_rate": 9.939320537261326e-07, + "loss": 0.0893, + "num_tokens": 291461028.0, + "reward": 1.11962890625, + "reward_std": 0.1981087028980255, + "rewards/accuracy_reward/mean": 0.166015625, + "rewards/accuracy_reward/std": 0.3724585771560669, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.95361328125, + "rewards/tag_count_reward/std": 0.13785336911678314, + "step": 431 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.072265625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 1048.94140625, + "completions/mean_terminated_length": 971.1199340820312, + "completions/min_length": 41.0, + "completions/min_terminated_length": 41.0, + "epoch": 0.14747802338482546, + "grad_norm": 0.3985885679721832, + "kl": 0.14013671875, + "learning_rate": 9.93843995629499e-07, + "loss": 0.0741, + "num_tokens": 292083702.0, + "reward": 1.03857421875, + "reward_std": 0.20800724625587463, + "rewards/accuracy_reward/mean": 0.0947580635547638, + "rewards/accuracy_reward/std": 0.29317617416381836, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.94677734375, + "rewards/tag_count_reward/std": 0.15155541896820068, + "step": 432 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2025.0, + "completions/mean_length": 1123.486328125, + "completions/mean_terminated_length": 1027.846923828125, + "completions/min_length": 165.0, + "completions/min_terminated_length": 165.0, + "epoch": 0.14781940769821628, + "grad_norm": 0.6801993250846863, + "kl": 0.17626953125, + "learning_rate": 9.937553075774938e-07, + "loss": 0.1014, + "num_tokens": 292741615.0, + "reward": 1.0771484375, + "reward_std": 0.24764740467071533, + "rewards/accuracy_reward/mean": 0.140625, + "rewards/accuracy_reward/std": 0.3479743003845215, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9365234375, + "rewards/tag_count_reward/std": 0.15834276378154755, + "step": 433 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 1087.423828125, + "completions/mean_terminated_length": 1031.853271484375, + "completions/min_length": 243.0, + "completions/min_terminated_length": 243.0, + "epoch": 0.14816079201160706, + "grad_norm": 13.213711738586426, + "kl": 0.439697265625, + "learning_rate": 9.936659896959935e-07, + "loss": 0.0976, + "num_tokens": 293375336.0, + "reward": 1.03515625, + "reward_std": 0.23112015426158905, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.943359375, + "rewards/tag_count_reward/std": 0.15313054621219635, + "step": 434 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 1145.2109375, + "completions/mean_terminated_length": 1038.7685546875, + "completions/min_length": 255.0, + "completions/min_terminated_length": 255.0, + "epoch": 0.14850217632499788, + "grad_norm": 0.8606828451156616, + "kl": 0.2158203125, + "learning_rate": 9.935760421117686e-07, + "loss": 0.0989, + "num_tokens": 294041028.0, + "reward": 1.04541015625, + "reward_std": 0.2330261766910553, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.93994140625, + "rewards/tag_count_reward/std": 0.17467160522937775, + "step": 435 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.060546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2009.0, + "completions/mean_length": 1060.1171875, + "completions/mean_terminated_length": 996.4490966796875, + "completions/min_length": 196.0, + "completions/min_terminated_length": 196.0, + "epoch": 0.14884356063838866, + "grad_norm": 1.6122773885726929, + "kl": 0.2325439453125, + "learning_rate": 9.93485464952483e-07, + "loss": 0.0901, + "num_tokens": 294661440.0, + "reward": 1.1298828125, + "reward_std": 0.26418039202690125, + "rewards/accuracy_reward/mean": 0.173828125, + "rewards/accuracy_reward/std": 0.3793322443962097, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9560546875, + "rewards/tag_count_reward/std": 0.1346244364976883, + "step": 436 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 968.67578125, + "completions/mean_terminated_length": 922.5133056640625, + "completions/min_length": 224.0, + "completions/min_terminated_length": 224.0, + "epoch": 0.14918494495177947, + "grad_norm": 18.031034469604492, + "kl": 0.348876953125, + "learning_rate": 9.933942583466944e-07, + "loss": 0.0767, + "num_tokens": 295239338.0, + "reward": 1.09326171875, + "reward_std": 0.1853155493736267, + "rewards/accuracy_reward/mean": 0.1391129046678543, + "rewards/accuracy_reward/std": 0.34641367197036743, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.95849609375, + "rewards/tag_count_reward/std": 0.13674680888652802, + "step": 437 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.056640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 1049.7109375, + "completions/mean_terminated_length": 989.7722778320312, + "completions/min_length": 138.0, + "completions/min_terminated_length": 138.0, + "epoch": 0.14952632926517026, + "grad_norm": 134.8598175048828, + "kl": 1.400146484375, + "learning_rate": 9.93302422423854e-07, + "loss": 0.1738, + "num_tokens": 295856694.0, + "reward": 1.04345703125, + "reward_std": 0.24107421934604645, + "rewards/accuracy_reward/mean": 0.1015625, + "rewards/accuracy_reward/std": 0.30236753821372986, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.94189453125, + "rewards/tag_count_reward/std": 0.1710955947637558, + "step": 438 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.080078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2030.0, + "completions/mean_length": 1089.255859375, + "completions/mean_terminated_length": 1005.79833984375, + "completions/min_length": 212.0, + "completions/min_terminated_length": 212.0, + "epoch": 0.14986771357856107, + "grad_norm": 12.19244384765625, + "kl": 0.2711181640625, + "learning_rate": 9.93209957314306e-07, + "loss": 0.1123, + "num_tokens": 296485497.0, + "reward": 0.99951171875, + "reward_std": 0.21098214387893677, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.93701171875, + "rewards/tag_count_reward/std": 0.17007611691951752, + "step": 439 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 1030.8984375, + "completions/mean_terminated_length": 972.0578002929688, + "completions/min_length": 160.0, + "completions/min_terminated_length": 160.0, + "epoch": 0.15020909789195186, + "grad_norm": 1.0037941932678223, + "kl": 0.201171875, + "learning_rate": 9.93116863149288e-07, + "loss": 0.1055, + "num_tokens": 297094325.0, + "reward": 1.13818359375, + "reward_std": 0.266563355922699, + "rewards/accuracy_reward/mean": 0.181640625, + "rewards/accuracy_reward/std": 0.38592514395713806, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.95654296875, + "rewards/tag_count_reward/std": 0.1405578851699829, + "step": 440 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0703125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 1062.68359375, + "completions/mean_terminated_length": 988.1639404296875, + "completions/min_length": 250.0, + "completions/min_terminated_length": 250.0, + "epoch": 0.15055048220534267, + "grad_norm": 1.679003357887268, + "kl": 0.250244140625, + "learning_rate": 9.9302314006093e-07, + "loss": 0.119, + "num_tokens": 297706547.0, + "reward": 1.1123046875, + "reward_std": 0.26775506138801575, + "rewards/accuracy_reward/mean": 0.16015625, + "rewards/accuracy_reward/std": 0.3671095669269562, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9521484375, + "rewards/tag_count_reward/std": 0.14387451112270355, + "step": 441 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.056640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 1019.21484375, + "completions/mean_terminated_length": 957.4451293945312, + "completions/min_length": 101.0, + "completions/min_terminated_length": 101.0, + "epoch": 0.15089186651873346, + "grad_norm": 0.7562029361724854, + "kl": 0.166748046875, + "learning_rate": 9.929287881822545e-07, + "loss": 0.064, + "num_tokens": 298306657.0, + "reward": 1.19970703125, + "reward_std": 0.2773776650428772, + "rewards/accuracy_reward/mean": 0.25604838132858276, + "rewards/accuracy_reward/std": 0.43688949942588806, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.95166015625, + "rewards/tag_count_reward/std": 0.1539822816848755, + "step": 442 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.064453125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1959.0, + "completions/mean_length": 1037.333984375, + "completions/mean_terminated_length": 967.7056884765625, + "completions/min_length": 127.0, + "completions/min_terminated_length": 127.0, + "epoch": 0.15123325083212427, + "grad_norm": 8.234630584716797, + "kl": 0.207763671875, + "learning_rate": 9.92833807647177e-07, + "loss": 0.0942, + "num_tokens": 298920076.0, + "reward": 1.068359375, + "reward_std": 0.2526884377002716, + "rewards/accuracy_reward/mean": 0.123046875, + "rewards/accuracy_reward/std": 0.32881227135658264, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9453125, + "rewards/tag_count_reward/std": 0.16007427871227264, + "step": 443 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2021.0, + "completions/mean_length": 1056.375, + "completions/mean_terminated_length": 963.1453247070312, + "completions/min_length": 211.0, + "completions/min_terminated_length": 211.0, + "epoch": 0.15157463514551506, + "grad_norm": 1.3715955018997192, + "kl": 0.132080078125, + "learning_rate": 9.927381985905051e-07, + "loss": 0.0946, + "num_tokens": 299539148.0, + "reward": 1.06494140625, + "reward_std": 0.27978986501693726, + "rewards/accuracy_reward/mean": 0.13671875, + "rewards/accuracy_reward/std": 0.3438861668109894, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.92822265625, + "rewards/tag_count_reward/std": 0.18266178667545319, + "step": 444 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.064453125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2029.0, + "completions/mean_length": 1059.0078125, + "completions/mean_terminated_length": 990.8726806640625, + "completions/min_length": 229.0, + "completions/min_terminated_length": 229.0, + "epoch": 0.15191601945890587, + "grad_norm": 0.4454004466533661, + "kl": 0.131103515625, + "learning_rate": 9.92641961147938e-07, + "loss": 0.0505, + "num_tokens": 300158368.0, + "reward": 1.04443359375, + "reward_std": 0.23585447669029236, + "rewards/accuracy_reward/mean": 0.0947580635547638, + "rewards/accuracy_reward/std": 0.29317617416381836, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.95263671875, + "rewards/tag_count_reward/std": 0.13207614421844482, + "step": 445 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.060546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 1133.0625, + "completions/mean_terminated_length": 1074.0955810546875, + "completions/min_length": 232.0, + "completions/min_terminated_length": 232.0, + "epoch": 0.15225740377229666, + "grad_norm": 0.8461600542068481, + "kl": 0.1156005859375, + "learning_rate": 9.925450954560676e-07, + "loss": 0.0838, + "num_tokens": 300813520.0, + "reward": 1.0185546875, + "reward_std": 0.21500445902347565, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9462890625, + "rewards/tag_count_reward/std": 0.16267666220664978, + "step": 446 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07421875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 1076.810546875, + "completions/mean_terminated_length": 998.951416015625, + "completions/min_length": 232.0, + "completions/min_terminated_length": 232.0, + "epoch": 0.15259878808568747, + "grad_norm": 0.7099547386169434, + "kl": 0.14013671875, + "learning_rate": 9.924476016523765e-07, + "loss": 0.1158, + "num_tokens": 301436703.0, + "reward": 1.01171875, + "reward_std": 0.18262024223804474, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.953125, + "rewards/tag_count_reward/std": 0.1467188447713852, + "step": 447 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 1066.75, + "completions/mean_terminated_length": 974.4957885742188, + "completions/min_length": 185.0, + "completions/min_terminated_length": 185.0, + "epoch": 0.15294017239907826, + "grad_norm": 0.7692645788192749, + "kl": 0.123779296875, + "learning_rate": 9.9234947987524e-07, + "loss": 0.0775, + "num_tokens": 302060671.0, + "reward": 1.072265625, + "reward_std": 0.24299228191375732, + "rewards/accuracy_reward/mean": 0.14314515888690948, + "rewards/accuracy_reward/std": 0.35057440400123596, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.93359375, + "rewards/tag_count_reward/std": 0.1748199313879013, + "step": 448 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 1084.212890625, + "completions/mean_terminated_length": 1015.6589965820312, + "completions/min_length": 249.0, + "completions/min_terminated_length": 249.0, + "epoch": 0.15328155671246907, + "grad_norm": 1.9816856384277344, + "kl": 0.161865234375, + "learning_rate": 9.922507302639234e-07, + "loss": 0.1198, + "num_tokens": 302686556.0, + "reward": 1.02685546875, + "reward_std": 0.18609923124313354, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.95458984375, + "rewards/tag_count_reward/std": 0.14508728682994843, + "step": 449 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 987.177734375, + "completions/mean_terminated_length": 932.7207641601562, + "completions/min_length": 215.0, + "completions/min_terminated_length": 215.0, + "epoch": 0.15362294102585985, + "grad_norm": 0.7764598727226257, + "kl": 0.139892578125, + "learning_rate": 9.921513529585842e-07, + "loss": 0.0547, + "num_tokens": 303266807.0, + "reward": 1.15380859375, + "reward_std": 0.21334266662597656, + "rewards/accuracy_reward/mean": 0.193359375, + "rewards/accuracy_reward/std": 0.39531853795051575, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.96044921875, + "rewards/tag_count_reward/std": 0.12810981273651123, + "step": 450 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.060546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2041.0, + "completions/mean_length": 1081.927734375, + "completions/mean_terminated_length": 1019.665283203125, + "completions/min_length": 231.0, + "completions/min_terminated_length": 231.0, + "epoch": 0.15396432533925067, + "grad_norm": 43.58025360107422, + "kl": 1.106201171875, + "learning_rate": 9.920513481002698e-07, + "loss": 0.1273, + "num_tokens": 303902242.0, + "reward": 1.0888671875, + "reward_std": 0.24603061378002167, + "rewards/accuracy_reward/mean": 0.138671875, + "rewards/accuracy_reward/std": 0.34594178199768066, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.9482421875, + "rewards/tag_count_reward/std": 0.16330981254577637, + "step": 451 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 1040.08984375, + "completions/mean_terminated_length": 968.3974609375, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "epoch": 0.15430570965264145, + "grad_norm": 225.49293518066406, + "kl": 2.928466796875, + "learning_rate": 9.919507158309192e-07, + "loss": 0.2038, + "num_tokens": 304511056.0, + "reward": 1.0263671875, + "reward_std": 0.20071804523468018, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9521484375, + "rewards/tag_count_reward/std": 0.14888782799243927, + "step": 452 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 990.08984375, + "completions/mean_terminated_length": 949.3184204101562, + "completions/min_length": 218.0, + "completions/min_terminated_length": 218.0, + "epoch": 0.15464709396603227, + "grad_norm": 1.5338170528411865, + "kl": 0.320556640625, + "learning_rate": 9.918494562933614e-07, + "loss": 0.1019, + "num_tokens": 305103246.0, + "reward": 1.13720703125, + "reward_std": 0.18577462434768677, + "rewards/accuracy_reward/mean": 0.173828125, + "rewards/accuracy_reward/std": 0.3793322443962097, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.96337890625, + "rewards/tag_count_reward/std": 0.12216134369373322, + "step": 453 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2017.0, + "completions/mean_length": 987.21484375, + "completions/mean_terminated_length": 950.7838745117188, + "completions/min_length": 140.0, + "completions/min_terminated_length": 140.0, + "epoch": 0.15498847827942305, + "grad_norm": 4.054177284240723, + "kl": 0.359375, + "learning_rate": 9.917475696313157e-07, + "loss": 0.0613, + "num_tokens": 305682364.0, + "reward": 1.0966796875, + "reward_std": 0.21819844841957092, + "rewards/accuracy_reward/mean": 0.1391129046678543, + "rewards/accuracy_reward/std": 0.3464137017726898, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9619140625, + "rewards/tag_count_reward/std": 0.11499445885419846, + "step": 454 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.111328125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2009.0, + "completions/mean_length": 1219.708984375, + "completions/mean_terminated_length": 1115.945068359375, + "completions/min_length": 255.0, + "completions/min_terminated_length": 255.0, + "epoch": 0.15532986259281387, + "grad_norm": 842.3438720703125, + "kl": 11.1953125, + "learning_rate": 9.916450559893917e-07, + "loss": 0.55, + "num_tokens": 306388247.0, + "reward": 0.98046875, + "reward_std": 0.2435244619846344, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.919921875, + "rewards/tag_count_reward/std": 0.1932915896177292, + "step": 455 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1999.0, + "completions/mean_length": 1036.408203125, + "completions/mean_terminated_length": 986.65771484375, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.15567124690620465, + "grad_norm": 1.067717432975769, + "kl": 0.24267578125, + "learning_rate": 9.915419155130886e-07, + "loss": 0.0952, + "num_tokens": 307004200.0, + "reward": 1.08251953125, + "reward_std": 0.23473551869392395, + "rewards/accuracy_reward/mean": 0.1328125, + "rewards/accuracy_reward/std": 0.33970388770103455, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.94970703125, + "rewards/tag_count_reward/std": 0.1549411565065384, + "step": 456 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 987.5859375, + "completions/mean_terminated_length": 951.167724609375, + "completions/min_length": 207.0, + "completions/min_terminated_length": 207.0, + "epoch": 0.15601263121959547, + "grad_norm": 73.63131713867188, + "kl": 1.642578125, + "learning_rate": 9.914381483487957e-07, + "loss": 0.1524, + "num_tokens": 307580292.0, + "reward": 1.0546875, + "reward_std": 0.19174796342849731, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.962890625, + "rewards/tag_count_reward/std": 0.12740769982337952, + "step": 457 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 1013.505859375, + "completions/mean_terminated_length": 962.6290283203125, + "completions/min_length": 178.0, + "completions/min_terminated_length": 178.0, + "epoch": 0.15635401553298625, + "grad_norm": 0.8415966033935547, + "kl": 0.1641845703125, + "learning_rate": 9.913337546437912e-07, + "loss": 0.0852, + "num_tokens": 308176887.0, + "reward": 1.091796875, + "reward_std": 0.23131389915943146, + "rewards/accuracy_reward/mean": 0.142578125, + "rewards/accuracy_reward/std": 0.3499840497970581, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.94921875, + "rewards/tag_count_reward/std": 0.15118376910686493, + "step": 458 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.080078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2009.0, + "completions/mean_length": 1108.814453125, + "completions/mean_terminated_length": 1027.0594482421875, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "epoch": 0.15669539984637706, + "grad_norm": 1.0721087455749512, + "kl": 0.175048828125, + "learning_rate": 9.91228734546243e-07, + "loss": 0.0635, + "num_tokens": 308819912.0, + "reward": 1.13818359375, + "reward_std": 0.25097551941871643, + "rewards/accuracy_reward/mean": 0.1953125, + "rewards/accuracy_reward/std": 0.3968288004398346, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.94287109375, + "rewards/tag_count_reward/std": 0.1663554310798645, + "step": 459 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.072265625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2019.0, + "completions/mean_length": 1101.990234375, + "completions/mean_terminated_length": 1028.301025390625, + "completions/min_length": 187.0, + "completions/min_terminated_length": 187.0, + "epoch": 0.15703678415976785, + "grad_norm": 1.2053905725479126, + "kl": 0.21875, + "learning_rate": 9.911230882052082e-07, + "loss": 0.095, + "num_tokens": 309464019.0, + "reward": 1.00048828125, + "reward_std": 0.21932029724121094, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.93408203125, + "rewards/tag_count_reward/std": 0.17743425071239471, + "step": 460 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.064453125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2013.0, + "completions/mean_length": 1110.123046875, + "completions/mean_terminated_length": 1045.5093994140625, + "completions/min_length": 117.0, + "completions/min_terminated_length": 117.0, + "epoch": 0.15737816847315866, + "grad_norm": 4.203698635101318, + "kl": 0.2410888671875, + "learning_rate": 9.91016815770632e-07, + "loss": 0.1079, + "num_tokens": 310110258.0, + "reward": 1.044921875, + "reward_std": 0.23291827738285065, + "rewards/accuracy_reward/mean": 0.10080645233392715, + "rewards/accuracy_reward/std": 0.30137622356414795, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.947265625, + "rewards/tag_count_reward/std": 0.15293073654174805, + "step": 461 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 1089.384765625, + "completions/mean_terminated_length": 1025.4771728515625, + "completions/min_length": 267.0, + "completions/min_terminated_length": 267.0, + "epoch": 0.15771955278654945, + "grad_norm": 1.4528241157531738, + "kl": 0.2091064453125, + "learning_rate": 9.90909917393349e-07, + "loss": 0.1112, + "num_tokens": 310744663.0, + "reward": 1.07275390625, + "reward_std": 0.21886608004570007, + "rewards/accuracy_reward/mean": 0.123046875, + "rewards/accuracy_reward/std": 0.32881227135658264, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.94970703125, + "rewards/tag_count_reward/std": 0.1509426087141037, + "step": 462 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 1066.322265625, + "completions/mean_terminated_length": 1011.6721801757812, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.15806093709994026, + "grad_norm": 1.0756810903549194, + "kl": 0.197265625, + "learning_rate": 9.908023932250816e-07, + "loss": 0.0782, + "num_tokens": 311379004.0, + "reward": 1.07568359375, + "reward_std": 0.21483656764030457, + "rewards/accuracy_reward/mean": 0.126953125, + "rewards/accuracy_reward/std": 0.33324605226516724, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.94873046875, + "rewards/tag_count_reward/std": 0.1481568068265915, + "step": 463 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1999.0, + "completions/mean_length": 1004.3046875, + "completions/mean_terminated_length": 955.2147216796875, + "completions/min_length": 212.0, + "completions/min_terminated_length": 212.0, + "epoch": 0.15840232141333105, + "grad_norm": 0.9950715899467468, + "kl": 0.27783203125, + "learning_rate": 9.906942434184411e-07, + "loss": 0.0879, + "num_tokens": 311966824.0, + "reward": 1.06884765625, + "reward_std": 0.2228804975748062, + "rewards/accuracy_reward/mean": 0.115234375, + "rewards/accuracy_reward/std": 0.3196168541908264, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.95361328125, + "rewards/tag_count_reward/std": 0.14976051449775696, + "step": 464 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2031.0, + "completions/mean_length": 1064.376953125, + "completions/mean_terminated_length": 1030.595947265625, + "completions/min_length": 206.0, + "completions/min_terminated_length": 206.0, + "epoch": 0.15874370572672186, + "grad_norm": 47.52520751953125, + "kl": 1.08154296875, + "learning_rate": 9.90585468126926e-07, + "loss": 0.1194, + "num_tokens": 312585881.0, + "reward": 1.09619140625, + "reward_std": 0.19244486093521118, + "rewards/accuracy_reward/mean": 0.1328125, + "rewards/accuracy_reward/std": 0.33970388770103455, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.96337890625, + "rewards/tag_count_reward/std": 0.1211559996008873, + "step": 465 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1994.0, + "completions/mean_length": 1057.943359375, + "completions/mean_terminated_length": 1023.9414672851562, + "completions/min_length": 266.0, + "completions/min_terminated_length": 266.0, + "epoch": 0.15908509004011265, + "grad_norm": 2.6732664108276367, + "kl": 0.335205078125, + "learning_rate": 9.904760675049233e-07, + "loss": 0.0679, + "num_tokens": 313203164.0, + "reward": 1.05126953125, + "reward_std": 0.18792620301246643, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.96923828125, + "rewards/tag_count_reward/std": 0.10906112939119339, + "step": 466 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.060546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 1119.1484375, + "completions/mean_terminated_length": 1059.2847900390625, + "completions/min_length": 89.0, + "completions/min_terminated_length": 89.0, + "epoch": 0.15942647435350346, + "grad_norm": 3.407625436782837, + "kl": 0.3837890625, + "learning_rate": 9.903660417077069e-07, + "loss": 0.0881, + "num_tokens": 313853624.0, + "reward": 1.037109375, + "reward_std": 0.19400563836097717, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.95703125, + "rewards/tag_count_reward/std": 0.12748268246650696, + "step": 467 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1998.0, + "completions/mean_length": 991.5078125, + "completions/mean_terminated_length": 966.1520385742188, + "completions/min_length": 276.0, + "completions/min_terminated_length": 276.0, + "epoch": 0.15976785866689425, + "grad_norm": 0.8257997632026672, + "kl": 0.239990234375, + "learning_rate": 9.90255390891438e-07, + "loss": 0.0711, + "num_tokens": 314431452.0, + "reward": 1.017578125, + "reward_std": 0.14489471912384033, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.97265625, + "rewards/tag_count_reward/std": 0.11162548512220383, + "step": 468 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1992.0, + "completions/mean_length": 1054.896484375, + "completions/mean_terminated_length": 1020.7899169921875, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "epoch": 0.16010924298028506, + "grad_norm": 0.830088198184967, + "kl": 0.285888671875, + "learning_rate": 9.90144115213166e-07, + "loss": 0.0645, + "num_tokens": 315046679.0, + "reward": 1.033203125, + "reward_std": 0.1966622769832611, + "rewards/accuracy_reward/mean": 0.07661290466785431, + "rewards/accuracy_reward/std": 0.2662447690963745, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.958984375, + "rewards/tag_count_reward/std": 0.1337306946516037, + "step": 469 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 1020.796875, + "completions/mean_terminated_length": 979.0405883789062, + "completions/min_length": 228.0, + "completions/min_terminated_length": 228.0, + "epoch": 0.16045062729367585, + "grad_norm": 28.442277908325195, + "kl": 0.573486328125, + "learning_rate": 9.900322148308256e-07, + "loss": 0.1236, + "num_tokens": 315650047.0, + "reward": 1.0419921875, + "reward_std": 0.19466054439544678, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9619140625, + "rewards/tag_count_reward/std": 0.12615279853343964, + "step": 470 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 1060.900390625, + "completions/mean_terminated_length": 1022.8579711914062, + "completions/min_length": 265.0, + "completions/min_terminated_length": 265.0, + "epoch": 0.16079201160706666, + "grad_norm": 2.216592311859131, + "kl": 0.27783203125, + "learning_rate": 9.899196899032393e-07, + "loss": 0.093, + "num_tokens": 316262812.0, + "reward": 1.0986328125, + "reward_std": 0.253503680229187, + "rewards/accuracy_reward/mean": 0.1328125, + "rewards/accuracy_reward/std": 0.33970388770103455, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9658203125, + "rewards/tag_count_reward/std": 0.1203538104891777, + "step": 471 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 1004.7265625, + "completions/mean_terminated_length": 975.3975830078125, + "completions/min_length": 187.0, + "completions/min_terminated_length": 187.0, + "epoch": 0.16113339592045745, + "grad_norm": 34.538063049316406, + "kl": 0.72900390625, + "learning_rate": 9.898065405901156e-07, + "loss": 0.123, + "num_tokens": 316842944.0, + "reward": 1.0537109375, + "reward_std": 0.18075445294380188, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9716796875, + "rewards/tag_count_reward/std": 0.10458524525165558, + "step": 472 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2020.0, + "completions/mean_length": 1024.359375, + "completions/mean_terminated_length": 984.90869140625, + "completions/min_length": 224.0, + "completions/min_terminated_length": 224.0, + "epoch": 0.16147478023384826, + "grad_norm": 0.7149609923362732, + "kl": 0.164306640625, + "learning_rate": 9.896927670520495e-07, + "loss": 0.0898, + "num_tokens": 317446280.0, + "reward": 1.134765625, + "reward_std": 0.22232946753501892, + "rewards/accuracy_reward/mean": 0.166015625, + "rewards/accuracy_reward/std": 0.3724585771560669, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.96875, + "rewards/tag_count_reward/std": 0.11169394850730896, + "step": 473 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2030.0, + "completions/mean_length": 973.15625, + "completions/mean_terminated_length": 949.556884765625, + "completions/min_length": 148.0, + "completions/min_terminated_length": 148.0, + "epoch": 0.16181616454723904, + "grad_norm": 67.18488311767578, + "kl": 1.100830078125, + "learning_rate": 9.895783694505212e-07, + "loss": 0.0751, + "num_tokens": 318013240.0, + "reward": 1.09521484375, + "reward_std": 0.17843782901763916, + "rewards/accuracy_reward/mean": 0.125, + "rewards/accuracy_reward/std": 0.3310423493385315, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.97021484375, + "rewards/tag_count_reward/std": 0.11154734343290329, + "step": 474 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 1131.65625, + "completions/mean_terminated_length": 1117.1112060546875, + "completions/min_length": 210.0, + "completions/min_terminated_length": 210.0, + "epoch": 0.16215754886062986, + "grad_norm": 0.2888360023498535, + "kl": 0.1614990234375, + "learning_rate": 9.894633479478974e-07, + "loss": 0.0367, + "num_tokens": 318669240.0, + "reward": 1.05419921875, + "reward_std": 0.1976710706949234, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.96826171875, + "rewards/tag_count_reward/std": 0.11426402628421783, + "step": 475 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2011.0, + "completions/mean_length": 994.228515625, + "completions/mean_terminated_length": 935.5650024414062, + "completions/min_length": 8.0, + "completions/min_terminated_length": 8.0, + "epoch": 0.16249893317402064, + "grad_norm": 1.06907320022583, + "kl": 0.17431640625, + "learning_rate": 9.893477027074303e-07, + "loss": 0.0649, + "num_tokens": 319255485.0, + "reward": 1.13916015625, + "reward_std": 0.22496505081653595, + "rewards/accuracy_reward/mean": 0.17943547666072845, + "rewards/accuracy_reward/std": 0.3841039538383484, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.96533203125, + "rewards/tag_count_reward/std": 0.1197040006518364, + "step": 476 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 1008.46484375, + "completions/mean_terminated_length": 968.401611328125, + "completions/min_length": 217.0, + "completions/min_terminated_length": 217.0, + "epoch": 0.16284031748741146, + "grad_norm": 0.6685115098953247, + "kl": 0.1739501953125, + "learning_rate": 9.89231433893257e-07, + "loss": 0.067, + "num_tokens": 319849163.0, + "reward": 1.0595703125, + "reward_std": 0.2082090675830841, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9638671875, + "rewards/tag_count_reward/std": 0.12280593812465668, + "step": 477 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2015.0, + "completions/mean_length": 1101.3984375, + "completions/mean_terminated_length": 1058.89794921875, + "completions/min_length": 244.0, + "completions/min_terminated_length": 244.0, + "epoch": 0.16318170180080224, + "grad_norm": 1.3302628993988037, + "kl": 0.180908203125, + "learning_rate": 9.891145416703998e-07, + "loss": 0.0726, + "num_tokens": 320490119.0, + "reward": 1.05517578125, + "reward_std": 0.20020583271980286, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.96142578125, + "rewards/tag_count_reward/std": 0.12551778554916382, + "step": 478 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 1059.849609375, + "completions/mean_terminated_length": 1013.3721923828125, + "completions/min_length": 123.0, + "completions/min_terminated_length": 123.0, + "epoch": 0.16352308611419306, + "grad_norm": 1.8521498441696167, + "kl": 0.22265625, + "learning_rate": 9.889970262047658e-07, + "loss": 0.0861, + "num_tokens": 321106218.0, + "reward": 0.98583984375, + "reward_std": 0.18413066864013672, + "rewards/accuracy_reward/mean": 0.033203125, + "rewards/accuracy_reward/std": 0.17934183776378632, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.95263671875, + "rewards/tag_count_reward/std": 0.14103294909000397, + "step": 479 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 1031.974609375, + "completions/mean_terminated_length": 1011.735107421875, + "completions/min_length": 180.0, + "completions/min_terminated_length": 180.0, + "epoch": 0.16386447042758384, + "grad_norm": 3.3183910846710205, + "kl": 0.25537109375, + "learning_rate": 9.888788876631467e-07, + "loss": 0.092, + "num_tokens": 321712525.0, + "reward": 1.126953125, + "reward_std": 0.24029642343521118, + "rewards/accuracy_reward/mean": 0.16015625, + "rewards/accuracy_reward/std": 0.3671095669269562, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.966796875, + "rewards/tag_count_reward/std": 0.11960916966199875, + "step": 480 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2014.0, + "completions/mean_length": 1030.52734375, + "completions/mean_terminated_length": 989.1666259765625, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "epoch": 0.16420585474097465, + "grad_norm": 310.8680725097656, + "kl": 8.68359375, + "learning_rate": 9.887601262132187e-07, + "loss": 0.4131, + "num_tokens": 322314619.0, + "reward": 1.0546875, + "reward_std": 0.2168104648590088, + "rewards/accuracy_reward/mean": 0.10080645233392715, + "rewards/accuracy_reward/std": 0.30137622356414795, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.95703125, + "rewards/tag_count_reward/std": 0.13493992388248444, + "step": 481 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2016.0, + "completions/mean_length": 1054.1328125, + "completions/mean_terminated_length": 1005.2540283203125, + "completions/min_length": 227.0, + "completions/min_terminated_length": 227.0, + "epoch": 0.16454723905436544, + "grad_norm": 31.778573989868164, + "kl": 1.4140625, + "learning_rate": 9.88640742023542e-07, + "loss": 0.1303, + "num_tokens": 322938319.0, + "reward": 1.03369140625, + "reward_std": 0.20622281730175018, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.95166015625, + "rewards/tag_count_reward/std": 0.14665940403938293, + "step": 482 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 1053.232421875, + "completions/mean_terminated_length": 1008.5693359375, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.16488862336775625, + "grad_norm": 2.6055119037628174, + "kl": 0.40234375, + "learning_rate": 9.8852073526356e-07, + "loss": 0.1049, + "num_tokens": 323550070.0, + "reward": 1.00537109375, + "reward_std": 0.19062183797359467, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.94482421875, + "rewards/tag_count_reward/std": 0.16180720925331116, + "step": 483 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 956.6328125, + "completions/mean_terminated_length": 909.9552612304688, + "completions/min_length": 125.0, + "completions/min_terminated_length": 125.0, + "epoch": 0.16523000768114704, + "grad_norm": 303.0550842285156, + "kl": 10.84619140625, + "learning_rate": 9.884001061036013e-07, + "loss": 0.5225, + "num_tokens": 324120074.0, + "reward": 1.0517578125, + "reward_std": 0.21274060010910034, + "rewards/accuracy_reward/mean": 0.09879032522439957, + "rewards/accuracy_reward/std": 0.2986815273761749, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9560546875, + "rewards/tag_count_reward/std": 0.15090225636959076, + "step": 484 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1992.0, + "completions/mean_length": 1072.443359375, + "completions/mean_terminated_length": 980.7244262695312, + "completions/min_length": 159.0, + "completions/min_terminated_length": 159.0, + "epoch": 0.16557139199453785, + "grad_norm": 80.19758605957031, + "kl": 2.73095703125, + "learning_rate": 9.882788547148764e-07, + "loss": 0.2371, + "num_tokens": 324760365.0, + "reward": 1.09326171875, + "reward_std": 0.32172858715057373, + "rewards/accuracy_reward/mean": 0.1640625, + "rewards/accuracy_reward/std": 0.37069445848464966, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.92919921875, + "rewards/tag_count_reward/std": 0.18896137177944183, + "step": 485 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2006.0, + "completions/mean_length": 1032.154296875, + "completions/mean_terminated_length": 1001.4949340820312, + "completions/min_length": 99.0, + "completions/min_terminated_length": 99.0, + "epoch": 0.16591277630792864, + "grad_norm": 1.6599100828170776, + "kl": 0.298095703125, + "learning_rate": 9.881569812694795e-07, + "loss": 0.0698, + "num_tokens": 325370156.0, + "reward": 1.0576171875, + "reward_std": 0.20596779882907867, + "rewards/accuracy_reward/mean": 0.0947580635547638, + "rewards/accuracy_reward/std": 0.29317617416381836, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9658203125, + "rewards/tag_count_reward/std": 0.1203538104891777, + "step": 486 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 985.86328125, + "completions/mean_terminated_length": 949.3859252929688, + "completions/min_length": 155.0, + "completions/min_terminated_length": 155.0, + "epoch": 0.16625416062131945, + "grad_norm": 24.591007232666016, + "kl": 0.68408203125, + "learning_rate": 9.880344859403876e-07, + "loss": 0.1034, + "num_tokens": 325951638.0, + "reward": 1.06884765625, + "reward_std": 0.2342439591884613, + "rewards/accuracy_reward/mean": 0.11088709533214569, + "rewards/accuracy_reward/std": 0.3143092691898346, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.95947265625, + "rewards/tag_count_reward/std": 0.14315126836299896, + "step": 487 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1991.0, + "completions/mean_length": 1083.4609375, + "completions/mean_terminated_length": 1031.860107421875, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "epoch": 0.16659554493471024, + "grad_norm": 2.2207202911376953, + "kl": 0.44921875, + "learning_rate": 9.879113689014606e-07, + "loss": 0.0804, + "num_tokens": 326580738.0, + "reward": 1.046875, + "reward_std": 0.1927567422389984, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9609375, + "rewards/tag_count_reward/std": 0.12778215110301971, + "step": 488 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2025.0, + "completions/mean_length": 1048.501953125, + "completions/mean_terminated_length": 981.8687744140625, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.16693692924810105, + "grad_norm": 51.33022689819336, + "kl": 1.2646484375, + "learning_rate": 9.877876303274404e-07, + "loss": 0.1344, + "num_tokens": 327202627.0, + "reward": 1.0400390625, + "reward_std": 0.2218092828989029, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9482421875, + "rewards/tag_count_reward/std": 0.1548524796962738, + "step": 489 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1969.0, + "completions/mean_length": 982.30078125, + "completions/mean_terminated_length": 961.0717163085938, + "completions/min_length": 256.0, + "completions/min_terminated_length": 256.0, + "epoch": 0.16727831356149184, + "grad_norm": 3.332066774368286, + "kl": 0.2041015625, + "learning_rate": 9.876632703939517e-07, + "loss": 0.0525, + "num_tokens": 327783885.0, + "reward": 1.080078125, + "reward_std": 0.22514007985591888, + "rewards/accuracy_reward/mean": 0.11328125, + "rewards/accuracy_reward/std": 0.3172462284564972, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.966796875, + "rewards/tag_count_reward/std": 0.11858218908309937, + "step": 490 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1988.0, + "completions/mean_length": 959.12890625, + "completions/mean_terminated_length": 935.2215576171875, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.16761969787488265, + "grad_norm": 3.5415234565734863, + "kl": 0.304443359375, + "learning_rate": 9.875382892775e-07, + "loss": 0.0913, + "num_tokens": 328349167.0, + "reward": 1.0859375, + "reward_std": 0.22196674346923828, + "rewards/accuracy_reward/mean": 0.1171875, + "rewards/accuracy_reward/std": 0.32195815443992615, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.96875, + "rewards/tag_count_reward/std": 0.11386296153068542, + "step": 491 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1986.0, + "completions/mean_length": 1010.57421875, + "completions/mean_terminated_length": 974.9454956054688, + "completions/min_length": 220.0, + "completions/min_terminated_length": 220.0, + "epoch": 0.16796108218827344, + "grad_norm": 1.6188727617263794, + "kl": 0.315185546875, + "learning_rate": 9.874126871554738e-07, + "loss": 0.0675, + "num_tokens": 328941525.0, + "reward": 1.07421875, + "reward_std": 0.2316889762878418, + "rewards/accuracy_reward/mean": 0.130859375, + "rewards/accuracy_reward/std": 0.33757632970809937, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.943359375, + "rewards/tag_count_reward/std": 0.14826077222824097, + "step": 492 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2009.0, + "completions/mean_length": 1048.296875, + "completions/mean_terminated_length": 1026.3472900390625, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "epoch": 0.16830246650166425, + "grad_norm": 2.26784086227417, + "kl": 0.2509765625, + "learning_rate": 9.872864642061419e-07, + "loss": 0.0641, + "num_tokens": 329553245.0, + "reward": 1.08056640625, + "reward_std": 0.23452627658843994, + "rewards/accuracy_reward/mean": 0.125, + "rewards/accuracy_reward/std": 0.3310423493385315, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.95556640625, + "rewards/tag_count_reward/std": 0.13582131266593933, + "step": 493 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1994.0, + "completions/mean_length": 864.888671875, + "completions/mean_terminated_length": 846.1091918945312, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.16864385081505504, + "grad_norm": 4.473270416259766, + "kl": 0.31396484375, + "learning_rate": 9.87159620608655e-07, + "loss": 0.0337, + "num_tokens": 330064740.0, + "reward": 1.04345703125, + "reward_std": 0.17220209538936615, + "rewards/accuracy_reward/mean": 0.07661290466785431, + "rewards/accuracy_reward/std": 0.2662447690963745, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.96923828125, + "rewards/tag_count_reward/std": 0.11769144982099533, + "step": 494 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1980.0, + "completions/mean_length": 866.771484375, + "completions/mean_terminated_length": 862.1392822265625, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.16898523512844585, + "grad_norm": 1.921745777130127, + "kl": 0.212890625, + "learning_rate": 9.87032156543044e-07, + "loss": 0.0083, + "num_tokens": 330589647.0, + "reward": 1.0869140625, + "reward_std": 0.1914043426513672, + "rewards/accuracy_reward/mean": 0.11290322244167328, + "rewards/accuracy_reward/std": 0.3167939782142639, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9775390625, + "rewards/tag_count_reward/std": 0.08975613117218018, + "step": 495 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1946.0, + "completions/mean_length": 958.35546875, + "completions/mean_terminated_length": 927.7228393554688, + "completions/min_length": 139.0, + "completions/min_terminated_length": 139.0, + "epoch": 0.16932661944183663, + "grad_norm": 1.4758943319320679, + "kl": 0.21435546875, + "learning_rate": 9.869040721902213e-07, + "loss": 0.0664, + "num_tokens": 331151141.0, + "reward": 1.16357421875, + "reward_std": 0.24888169765472412, + "rewards/accuracy_reward/mean": 0.193359375, + "rewards/accuracy_reward/std": 0.39531853795051575, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.97021484375, + "rewards/tag_count_reward/std": 0.12301874905824661, + "step": 496 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1943.0, + "completions/mean_length": 967.48828125, + "completions/mean_terminated_length": 952.5109252929688, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.16966800375522745, + "grad_norm": 0.5236533284187317, + "kl": 0.15771484375, + "learning_rate": 9.86775367731979e-07, + "loss": 0.0074, + "num_tokens": 331723199.0, + "reward": 1.03759765625, + "reward_std": 0.17600062489509583, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.97509765625, + "rewards/tag_count_reward/std": 0.10009774565696716, + "step": 497 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1974.0, + "completions/mean_length": 909.521484375, + "completions/mean_terminated_length": 905.0569458007812, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.17000938806861823, + "grad_norm": 0.37404993176460266, + "kl": 0.140869140625, + "learning_rate": 9.866460433509893e-07, + "loss": 0.0024, + "num_tokens": 332269418.0, + "reward": 1.095703125, + "reward_std": 0.19775967299938202, + "rewards/accuracy_reward/mean": 0.115234375, + "rewards/accuracy_reward/std": 0.3196168541908264, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.98046875, + "rewards/tag_count_reward/std": 0.08769373595714569, + "step": 498 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 912.21875, + "completions/mean_terminated_length": 891.8966064453125, + "completions/min_length": 233.0, + "completions/min_terminated_length": 233.0, + "epoch": 0.17035077238200905, + "grad_norm": 2.436751127243042, + "kl": 0.2852783203125, + "learning_rate": 9.865160992308047e-07, + "loss": 0.0769, + "num_tokens": 332812362.0, + "reward": 1.08544921875, + "reward_std": 0.2063451111316681, + "rewards/accuracy_reward/mean": 0.109375, + "rewards/accuracy_reward/std": 0.31241437792778015, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.97607421875, + "rewards/tag_count_reward/std": 0.10392878204584122, + "step": 499 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.009765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2025.0, + "completions/mean_length": 864.638671875, + "completions/mean_terminated_length": 852.9684448242188, + "completions/min_length": 161.0, + "completions/min_terminated_length": 161.0, + "epoch": 0.17069215669539983, + "grad_norm": 4.616459369659424, + "kl": 0.337158203125, + "learning_rate": 9.863855355558573e-07, + "loss": 0.0282, + "num_tokens": 333329649.0, + "reward": 1.13720703125, + "reward_std": 0.22087720036506653, + "rewards/accuracy_reward/mean": 0.162109375, + "rewards/accuracy_reward/std": 0.3689115643501282, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.97509765625, + "rewards/tag_count_reward/std": 0.10009774565696716, + "step": 500 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 994.232421875, + "completions/mean_terminated_length": 971.0958251953125, + "completions/min_length": 205.0, + "completions/min_terminated_length": 205.0, + "epoch": 0.17103354100879065, + "grad_norm": 1.0512064695358276, + "kl": 0.1864013671875, + "learning_rate": 9.862543525114582e-07, + "loss": 0.028, + "num_tokens": 333918568.0, + "reward": 1.1123046875, + "reward_std": 0.20634591579437256, + "rewards/accuracy_reward/mean": 0.14453125, + "rewards/accuracy_reward/std": 0.35197147727012634, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9677734375, + "rewards/tag_count_reward/std": 0.1157233864068985, + "step": 501 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.009765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 885.63671875, + "completions/mean_terminated_length": 874.173583984375, + "completions/min_length": 129.0, + "completions/min_terminated_length": 129.0, + "epoch": 0.17137492532218146, + "grad_norm": 0.5130018591880798, + "kl": 0.129638671875, + "learning_rate": 9.861225502837976e-07, + "loss": 0.0264, + "num_tokens": 334445566.0, + "reward": 1.08935546875, + "reward_std": 0.1409814953804016, + "rewards/accuracy_reward/mean": 0.111328125, + "rewards/accuracy_reward/std": 0.31484565138816833, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.97802734375, + "rewards/tag_count_reward/std": 0.0878121480345726, + "step": 502 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2004.0, + "completions/mean_length": 983.76953125, + "completions/mean_terminated_length": 969.017822265625, + "completions/min_length": 84.0, + "completions/min_terminated_length": 84.0, + "epoch": 0.17171630963557225, + "grad_norm": 0.9352298378944397, + "kl": 0.1884765625, + "learning_rate": 9.859901290599448e-07, + "loss": 0.0431, + "num_tokens": 335032824.0, + "reward": 1.111328125, + "reward_std": 0.21735718846321106, + "rewards/accuracy_reward/mean": 0.130859375, + "rewards/accuracy_reward/std": 0.33757632970809937, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.98046875, + "rewards/tag_count_reward/std": 0.09044018387794495, + "step": 503 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1993.0, + "completions/mean_length": 925.0859375, + "completions/mean_terminated_length": 893.51806640625, + "completions/min_length": 222.0, + "completions/min_terminated_length": 222.0, + "epoch": 0.17205769394896306, + "grad_norm": 2.7850639820098877, + "kl": 0.2857666015625, + "learning_rate": 9.858570890278475e-07, + "loss": 0.0822, + "num_tokens": 335584852.0, + "reward": 1.08203125, + "reward_std": 0.16623443365097046, + "rewards/accuracy_reward/mean": 0.11088709533214569, + "rewards/accuracy_reward/std": 0.3143092691898346, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.974609375, + "rewards/tag_count_reward/std": 0.11208678781986237, + "step": 504 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 910.5546875, + "completions/mean_terminated_length": 887.8964233398438, + "completions/min_length": 143.0, + "completions/min_terminated_length": 143.0, + "epoch": 0.17239907826235384, + "grad_norm": 1.236697793006897, + "kl": 0.231201171875, + "learning_rate": 9.857234303763317e-07, + "loss": 0.0535, + "num_tokens": 336125184.0, + "reward": 1.1552734375, + "reward_std": 0.24600103497505188, + "rewards/accuracy_reward/mean": 0.181640625, + "rewards/accuracy_reward/std": 0.38592514395713806, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9736328125, + "rewards/tag_count_reward/std": 0.09910601377487183, + "step": 505 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2002.0, + "completions/mean_length": 961.521484375, + "completions/mean_terminated_length": 933.2164306640625, + "completions/min_length": 215.0, + "completions/min_terminated_length": 215.0, + "epoch": 0.17274046257574466, + "grad_norm": 2.1265530586242676, + "kl": 0.252685546875, + "learning_rate": 9.855891532951015e-07, + "loss": 0.061, + "num_tokens": 336696107.0, + "reward": 1.12548828125, + "reward_std": 0.24798564612865448, + "rewards/accuracy_reward/mean": 0.158203125, + "rewards/accuracy_reward/std": 0.36528825759887695, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.96728515625, + "rewards/tag_count_reward/std": 0.1202535331249237, + "step": 506 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2030.0, + "completions/mean_length": 987.826171875, + "completions/mean_terminated_length": 937.9611206054688, + "completions/min_length": 211.0, + "completions/min_terminated_length": 211.0, + "epoch": 0.17308184688913544, + "grad_norm": 2.6731009483337402, + "kl": 0.27392578125, + "learning_rate": 9.854542579747383e-07, + "loss": 0.0921, + "num_tokens": 337282306.0, + "reward": 1.083984375, + "reward_std": 0.20409680902957916, + "rewards/accuracy_reward/mean": 0.123046875, + "rewards/accuracy_reward/std": 0.32881227135658264, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9609375, + "rewards/tag_count_reward/std": 0.14054512977600098, + "step": 507 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1937.0, + "completions/mean_length": 967.68359375, + "completions/mean_terminated_length": 939.5390625, + "completions/min_length": 194.0, + "completions/min_terminated_length": 194.0, + "epoch": 0.17342323120252626, + "grad_norm": 1.2340688705444336, + "kl": 0.247314453125, + "learning_rate": 9.853187446067019e-07, + "loss": 0.0591, + "num_tokens": 337859648.0, + "reward": 1.04248046875, + "reward_std": 0.188123419880867, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.97021484375, + "rewards/tag_count_reward/std": 0.11585026234388351, + "step": 508 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.068359375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2016.0, + "completions/mean_length": 1024.251953125, + "completions/mean_terminated_length": 949.1341552734375, + "completions/min_length": 275.0, + "completions/min_terminated_length": 275.0, + "epoch": 0.17376461551591704, + "grad_norm": 3.208914279937744, + "kl": 0.501953125, + "learning_rate": 9.85182613383328e-07, + "loss": 0.1623, + "num_tokens": 338471009.0, + "reward": 1.00146484375, + "reward_std": 0.2128455489873886, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.93896484375, + "rewards/tag_count_reward/std": 0.17780545353889465, + "step": 509 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.076171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1975.0, + "completions/mean_length": 1120.59765625, + "completions/mean_terminated_length": 1044.131103515625, + "completions/min_length": 227.0, + "completions/min_terminated_length": 227.0, + "epoch": 0.17410599982930786, + "grad_norm": 7.5583696365356445, + "kl": 0.53369140625, + "learning_rate": 9.850458644978307e-07, + "loss": 0.0999, + "num_tokens": 339118275.0, + "reward": 1.029296875, + "reward_std": 0.23589910566806793, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.935546875, + "rewards/tag_count_reward/std": 0.17968250811100006, + "step": 510 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 973.658203125, + "completions/mean_terminated_length": 902.0354614257812, + "completions/min_length": 256.0, + "completions/min_terminated_length": 256.0, + "epoch": 0.17444738414269864, + "grad_norm": 5.508956432342529, + "kl": 0.451171875, + "learning_rate": 9.849084981442997e-07, + "loss": 0.1656, + "num_tokens": 339696084.0, + "reward": 1.0224609375, + "reward_std": 0.1929110884666443, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9462890625, + "rewards/tag_count_reward/std": 0.16712692379951477, + "step": 511 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1956.0, + "completions/mean_length": 979.482421875, + "completions/mean_terminated_length": 903.4790649414062, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.17478876845608946, + "grad_norm": 3.7685906887054443, + "kl": 0.2861328125, + "learning_rate": 9.847705145177013e-07, + "loss": 0.1415, + "num_tokens": 340273419.0, + "reward": 0.98583984375, + "reward_std": 0.19239750504493713, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.94482421875, + "rewards/tag_count_reward/std": 0.1699187308549881, + "step": 512 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.111328125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1935.0, + "completions/mean_length": 1070.310546875, + "completions/mean_terminated_length": 947.830810546875, + "completions/min_length": 221.0, + "completions/min_terminated_length": 221.0, + "epoch": 0.17513015276948024, + "grad_norm": 1.9535354375839233, + "kl": 0.4921875, + "learning_rate": 9.84631913813878e-07, + "loss": 0.1717, + "num_tokens": 340905754.0, + "reward": 0.98828125, + "reward_std": 0.2464141696691513, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9140625, + "rewards/tag_count_reward/std": 0.21487605571746826, + "step": 513 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08984375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1963.0, + "completions/mean_length": 975.24609375, + "completions/mean_terminated_length": 869.3519287109375, + "completions/min_length": 181.0, + "completions/min_terminated_length": 181.0, + "epoch": 0.17547153708287105, + "grad_norm": 3.652279853820801, + "kl": 0.43310546875, + "learning_rate": 9.844926962295487e-07, + "loss": 0.1758, + "num_tokens": 341484072.0, + "reward": 1.0634765625, + "reward_std": 0.2978646457195282, + "rewards/accuracy_reward/mean": 0.130859375, + "rewards/accuracy_reward/std": 0.33757632970809937, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9326171875, + "rewards/tag_count_reward/std": 0.19308137893676758, + "step": 514 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.083984375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1989.0, + "completions/mean_length": 974.666015625, + "completions/mean_terminated_length": 876.2579956054688, + "completions/min_length": 237.0, + "completions/min_terminated_length": 237.0, + "epoch": 0.17581292139626184, + "grad_norm": 3.9282784461975098, + "kl": 0.5498046875, + "learning_rate": 9.843528619623068e-07, + "loss": 0.1777, + "num_tokens": 342060589.0, + "reward": 1.0673828125, + "reward_std": 0.28128862380981445, + "rewards/accuracy_reward/mean": 0.1328125, + "rewards/accuracy_reward/std": 0.33970388770103455, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9345703125, + "rewards/tag_count_reward/std": 0.1892828494310379, + "step": 515 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.123046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2022.0, + "completions/mean_length": 1088.9453125, + "completions/mean_terminated_length": 954.378662109375, + "completions/min_length": 295.0, + "completions/min_terminated_length": 295.0, + "epoch": 0.17615430570965265, + "grad_norm": 10.424691200256348, + "kl": 0.84375, + "learning_rate": 9.842124112106214e-07, + "loss": 0.2623, + "num_tokens": 342701937.0, + "reward": 1.0419921875, + "reward_std": 0.294842392206192, + "rewards/accuracy_reward/mean": 0.130859375, + "rewards/accuracy_reward/std": 0.33757632970809937, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9111328125, + "rewards/tag_count_reward/std": 0.2142503559589386, + "step": 516 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.150390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2024.0, + "completions/mean_length": 1101.529296875, + "completions/mean_terminated_length": 933.9931030273438, + "completions/min_length": 230.0, + "completions/min_terminated_length": 230.0, + "epoch": 0.17649569002304344, + "grad_norm": 197.44442749023438, + "kl": 2.42919921875, + "learning_rate": 9.84071344173837e-07, + "loss": 0.2986, + "num_tokens": 343343216.0, + "reward": 0.97265625, + "reward_std": 0.2862517833709717, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.89453125, + "rewards/tag_count_reward/std": 0.23431077599525452, + "step": 517 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2007.0, + "completions/mean_length": 1096.52734375, + "completions/mean_terminated_length": 925.5253295898438, + "completions/min_length": 147.0, + "completions/min_terminated_length": 147.0, + "epoch": 0.17683707433643425, + "grad_norm": 5.872008323669434, + "kl": 0.8203125, + "learning_rate": 9.839296610521723e-07, + "loss": 0.2026, + "num_tokens": 343989006.0, + "reward": 0.95263671875, + "reward_std": 0.2677186131477356, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.89208984375, + "rewards/tag_count_reward/std": 0.23657894134521484, + "step": 518 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12890625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1954.0, + "completions/mean_length": 1106.880859375, + "completions/mean_terminated_length": 967.6121826171875, + "completions/min_length": 218.0, + "completions/min_terminated_length": 218.0, + "epoch": 0.17717845864982504, + "grad_norm": 7.19550895690918, + "kl": 0.6943359375, + "learning_rate": 9.837873620467203e-07, + "loss": 0.1934, + "num_tokens": 344637665.0, + "reward": 0.98095703125, + "reward_std": 0.2884325683116913, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.90087890625, + "rewards/tag_count_reward/std": 0.2294684797525406, + "step": 519 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11328125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2031.0, + "completions/mean_length": 1099.63671875, + "completions/mean_terminated_length": 978.4801635742188, + "completions/min_length": 200.0, + "completions/min_terminated_length": 200.0, + "epoch": 0.17751984296321585, + "grad_norm": 1.9427558183670044, + "kl": 0.462890625, + "learning_rate": 9.836444473594488e-07, + "loss": 0.1731, + "num_tokens": 345280039.0, + "reward": 0.97802734375, + "reward_std": 0.24953964352607727, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.91162109375, + "rewards/tag_count_reward/std": 0.21244709193706512, + "step": 520 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1993.0, + "completions/mean_length": 935.283203125, + "completions/mean_terminated_length": 830.6688232421875, + "completions/min_length": 201.0, + "completions/min_terminated_length": 201.0, + "epoch": 0.17786122727660664, + "grad_norm": 4.585904121398926, + "kl": 0.41796875, + "learning_rate": 9.83500917193199e-07, + "loss": 0.189, + "num_tokens": 345842104.0, + "reward": 1.01220703125, + "reward_std": 0.2607251703739166, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.93212890625, + "rewards/tag_count_reward/std": 0.18939071893692017, + "step": 521 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.138671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1953.0, + "completions/mean_length": 1115.462890625, + "completions/mean_terminated_length": 965.3265380859375, + "completions/min_length": 204.0, + "completions/min_terminated_length": 204.0, + "epoch": 0.17820261158999745, + "grad_norm": 3.002474546432495, + "kl": 0.43408203125, + "learning_rate": 9.833567717516856e-07, + "loss": 0.1894, + "num_tokens": 346490661.0, + "reward": 0.95751953125, + "reward_std": 0.25489163398742676, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.89697265625, + "rewards/tag_count_reward/std": 0.2309378832578659, + "step": 522 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 1051.08984375, + "completions/mean_terminated_length": 943.1991577148438, + "completions/min_length": 229.0, + "completions/min_terminated_length": 229.0, + "epoch": 0.17854399590338824, + "grad_norm": 1.89009428024292, + "kl": 0.46484375, + "learning_rate": 9.832120112394969e-07, + "loss": 0.1646, + "num_tokens": 347102707.0, + "reward": 0.97509765625, + "reward_std": 0.18446403741836548, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.93408203125, + "rewards/tag_count_reward/std": 0.18486134707927704, + "step": 523 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12890625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 1040.197265625, + "completions/mean_terminated_length": 891.0606079101562, + "completions/min_length": 198.0, + "completions/min_terminated_length": 198.0, + "epoch": 0.17888538021677905, + "grad_norm": 10.38375473022461, + "kl": 0.623779296875, + "learning_rate": 9.830666358620936e-07, + "loss": 0.1913, + "num_tokens": 347709880.0, + "reward": 1.064453125, + "reward_std": 0.3105708956718445, + "rewards/accuracy_reward/mean": 0.154296875, + "rewards/accuracy_reward/std": 0.36158639192581177, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.91015625, + "rewards/tag_count_reward/std": 0.21836963295936584, + "step": 524 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1997.0, + "completions/mean_length": 1002.6796875, + "completions/mean_terminated_length": 894.5430908203125, + "completions/min_length": 248.0, + "completions/min_terminated_length": 248.0, + "epoch": 0.17922676453016984, + "grad_norm": 3.3788344860076904, + "kl": 0.5263671875, + "learning_rate": 9.829206458258097e-07, + "loss": 0.1746, + "num_tokens": 348304372.0, + "reward": 1.041015625, + "reward_std": 0.23436088860034943, + "rewards/accuracy_reward/mean": 0.11328125, + "rewards/accuracy_reward/std": 0.3172462284564972, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.927734375, + "rewards/tag_count_reward/std": 0.19635210931301117, + "step": 525 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.107421875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2031.0, + "completions/mean_length": 1075.69921875, + "completions/mean_terminated_length": 958.6827392578125, + "completions/min_length": 220.0, + "completions/min_terminated_length": 220.0, + "epoch": 0.17956814884356065, + "grad_norm": 19.488237380981445, + "kl": 0.97705078125, + "learning_rate": 9.827740413378513e-07, + "loss": 0.1829, + "num_tokens": 348933850.0, + "reward": 1.02294921875, + "reward_std": 0.2862394452095032, + "rewards/accuracy_reward/mean": 0.107421875, + "rewards/accuracy_reward/std": 0.30995169281959534, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.91552734375, + "rewards/tag_count_reward/std": 0.21517324447631836, + "step": 526 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.103515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2000.0, + "completions/mean_length": 1089.953125, + "completions/mean_terminated_length": 979.3289794921875, + "completions/min_length": 312.0, + "completions/min_terminated_length": 312.0, + "epoch": 0.17990953315695143, + "grad_norm": 2.0587828159332275, + "kl": 0.36572265625, + "learning_rate": 9.826268226062967e-07, + "loss": 0.1547, + "num_tokens": 349574642.0, + "reward": 1.0166015625, + "reward_std": 0.26578330993652344, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9267578125, + "rewards/tag_count_reward/std": 0.19785255193710327, + "step": 527 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2016.0, + "completions/mean_length": 962.0703125, + "completions/mean_terminated_length": 899.2479248046875, + "completions/min_length": 255.0, + "completions/min_terminated_length": 255.0, + "epoch": 0.18025091747034225, + "grad_norm": 18.152191162109375, + "kl": 0.585205078125, + "learning_rate": 9.82478989840096e-07, + "loss": 0.1541, + "num_tokens": 350138598.0, + "reward": 1.10009765625, + "reward_std": 0.1812194138765335, + "rewards/accuracy_reward/mean": 0.140625, + "rewards/accuracy_reward/std": 0.3479743003845215, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.95947265625, + "rewards/tag_count_reward/std": 0.15145450830459595, + "step": 528 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08984375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 1021.46875, + "completions/mean_terminated_length": 920.1373291015625, + "completions/min_length": 195.0, + "completions/min_terminated_length": 195.0, + "epoch": 0.18059230178373303, + "grad_norm": 5.078516483306885, + "kl": 0.38037109375, + "learning_rate": 9.823305432490705e-07, + "loss": 0.1617, + "num_tokens": 350734054.0, + "reward": 1.02099609375, + "reward_std": 0.24846667051315308, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.92724609375, + "rewards/tag_count_reward/std": 0.19895724952220917, + "step": 529 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.134765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 1131.94140625, + "completions/mean_terminated_length": 989.2596435546875, + "completions/min_length": 283.0, + "completions/min_terminated_length": 283.0, + "epoch": 0.18093368609712385, + "grad_norm": 5.4803643226623535, + "kl": 0.34765625, + "learning_rate": 9.821814830439133e-07, + "loss": 0.1819, + "num_tokens": 351392728.0, + "reward": 0.98876953125, + "reward_std": 0.28743964433670044, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.89892578125, + "rewards/tag_count_reward/std": 0.2390739917755127, + "step": 530 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.076171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 952.357421875, + "completions/mean_terminated_length": 862.01904296875, + "completions/min_length": 147.0, + "completions/min_terminated_length": 147.0, + "epoch": 0.18127507041051463, + "grad_norm": 6.403390407562256, + "kl": 0.372802734375, + "learning_rate": 9.820318094361883e-07, + "loss": 0.1543, + "num_tokens": 351960927.0, + "reward": 1.01171875, + "reward_std": 0.2463240623474121, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9296875, + "rewards/tag_count_reward/std": 0.19829878211021423, + "step": 531 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1987.0, + "completions/mean_length": 1078.537109375, + "completions/mean_terminated_length": 978.2478637695312, + "completions/min_length": 178.0, + "completions/min_terminated_length": 178.0, + "epoch": 0.18161645472390545, + "grad_norm": 6.348297595977783, + "kl": 0.53515625, + "learning_rate": 9.81881522638329e-07, + "loss": 0.2102, + "num_tokens": 352597138.0, + "reward": 1.02978515625, + "reward_std": 0.2722817659378052, + "rewards/accuracy_reward/mean": 0.109375, + "rewards/accuracy_reward/std": 0.31241437792778015, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.92041015625, + "rewards/tag_count_reward/std": 0.21247407793998718, + "step": 532 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.16015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1977.0, + "completions/mean_length": 1175.853515625, + "completions/mean_terminated_length": 1009.5372314453125, + "completions/min_length": 189.0, + "completions/min_terminated_length": 189.0, + "epoch": 0.18195783903729623, + "grad_norm": 3.549447536468506, + "kl": 0.7392578125, + "learning_rate": 9.817306228636411e-07, + "loss": 0.2242, + "num_tokens": 353274087.0, + "reward": 0.94091796875, + "reward_std": 0.31294119358062744, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.87255859375, + "rewards/tag_count_reward/std": 0.2565080523490906, + "step": 533 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.123046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 1033.978515625, + "completions/mean_terminated_length": 891.6993408203125, + "completions/min_length": 188.0, + "completions/min_terminated_length": 188.0, + "epoch": 0.18229922335068705, + "grad_norm": 69.35165405273438, + "kl": 1.7158203125, + "learning_rate": 9.815791103262981e-07, + "loss": 0.2082, + "num_tokens": 353872972.0, + "reward": 1.017578125, + "reward_std": 0.3158687949180603, + "rewards/accuracy_reward/mean": 0.109375, + "rewards/accuracy_reward/std": 0.31241437792778015, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.908203125, + "rewards/tag_count_reward/std": 0.22852177917957306, + "step": 534 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.134765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2013.0, + "completions/mean_length": 1086.748046875, + "completions/mean_terminated_length": 937.027099609375, + "completions/min_length": 314.0, + "completions/min_terminated_length": 314.0, + "epoch": 0.18264060766407783, + "grad_norm": 26.75957679748535, + "kl": 1.73095703125, + "learning_rate": 9.814269852413453e-07, + "loss": 0.2815, + "num_tokens": 354504011.0, + "reward": 1.060546875, + "reward_std": 0.3185046911239624, + "rewards/accuracy_reward/mean": 0.1713709682226181, + "rewards/accuracy_reward/std": 0.3772132694721222, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.89453125, + "rewards/tag_count_reward/std": 0.24049316346645355, + "step": 535 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 1036.91796875, + "completions/mean_terminated_length": 902.7035522460938, + "completions/min_length": 215.0, + "completions/min_terminated_length": 215.0, + "epoch": 0.18298199197746864, + "grad_norm": 10.085936546325684, + "kl": 0.70849609375, + "learning_rate": 9.812742478246957e-07, + "loss": 0.2268, + "num_tokens": 355112705.0, + "reward": 0.99169921875, + "reward_std": 0.2692239582538605, + "rewards/accuracy_reward/mean": 0.08467742055654526, + "rewards/accuracy_reward/std": 0.278682142496109, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.90966796875, + "rewards/tag_count_reward/std": 0.2272297441959381, + "step": 536 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.177734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1936.0, + "completions/mean_length": 1110.74609375, + "completions/mean_terminated_length": 908.1567993164062, + "completions/min_length": 156.0, + "completions/min_terminated_length": 156.0, + "epoch": 0.18332337629085943, + "grad_norm": 110.41934204101562, + "kl": 2.91796875, + "learning_rate": 9.811208982931327e-07, + "loss": 0.373, + "num_tokens": 355753231.0, + "reward": 0.94580078125, + "reward_std": 0.2905876040458679, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.87939453125, + "rewards/tag_count_reward/std": 0.2550473213195801, + "step": 537 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.130859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1964.0, + "completions/mean_length": 1066.75, + "completions/mean_terminated_length": 919.01123046875, + "completions/min_length": 264.0, + "completions/min_terminated_length": 264.0, + "epoch": 0.18366476060425024, + "grad_norm": 70.96749114990234, + "kl": 4.224609375, + "learning_rate": 9.809669368643075e-07, + "loss": 0.3807, + "num_tokens": 356369455.0, + "reward": 1.0087890625, + "reward_std": 0.3199254274368286, + "rewards/accuracy_reward/mean": 0.109375, + "rewards/accuracy_reward/std": 0.31241437792778015, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8994140625, + "rewards/tag_count_reward/std": 0.23953568935394287, + "step": 538 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1930.0, + "completions/mean_length": 1183.3671875, + "completions/mean_terminated_length": 952.2277221679688, + "completions/min_length": 212.0, + "completions/min_terminated_length": 212.0, + "epoch": 0.18400614491764103, + "grad_norm": 609.8837890625, + "kl": 26.34375, + "learning_rate": 9.808123637567406e-07, + "loss": 1.2569, + "num_tokens": 357051515.0, + "reward": 0.9091796875, + "reward_std": 0.3217463791370392, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8447265625, + "rewards/tag_count_reward/std": 0.28594186902046204, + "step": 539 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.181640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2013.0, + "completions/mean_length": 1120.263671875, + "completions/mean_terminated_length": 914.3460693359375, + "completions/min_length": 246.0, + "completions/min_terminated_length": 246.0, + "epoch": 0.18434752923103184, + "grad_norm": 276.126220703125, + "kl": 10.609375, + "learning_rate": 9.806571791898196e-07, + "loss": 0.6705, + "num_tokens": 357698066.0, + "reward": 0.92431640625, + "reward_std": 0.2812725007534027, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.86962890625, + "rewards/tag_count_reward/std": 0.2690318524837494, + "step": 540 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.19921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2016.0, + "completions/mean_length": 1167.890625, + "completions/mean_terminated_length": 948.9365844726562, + "completions/min_length": 128.0, + "completions/min_terminated_length": 128.0, + "epoch": 0.18468891354442263, + "grad_norm": 28.768630981445312, + "kl": 2.984375, + "learning_rate": 9.805013833838014e-07, + "loss": 0.3308, + "num_tokens": 358376490.0, + "reward": 0.96142578125, + "reward_std": 0.2962723672389984, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.86767578125, + "rewards/tag_count_reward/std": 0.26207634806632996, + "step": 541 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1966.0, + "completions/mean_length": 986.94921875, + "completions/mean_terminated_length": 835.3705444335938, + "completions/min_length": 142.0, + "completions/min_terminated_length": 142.0, + "epoch": 0.18503029785781344, + "grad_norm": 17.48018455505371, + "kl": 1.12109375, + "learning_rate": 9.80344976559809e-07, + "loss": 0.1957, + "num_tokens": 358964896.0, + "reward": 1.05078125, + "reward_std": 0.2224898636341095, + "rewards/accuracy_reward/mean": 0.13671875, + "rewards/accuracy_reward/std": 0.3438861668109894, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9140625, + "rewards/tag_count_reward/std": 0.21770349144935608, + "step": 542 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.158203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 1151.96875, + "completions/mean_terminated_length": 983.5730590820312, + "completions/min_length": 75.0, + "completions/min_terminated_length": 75.0, + "epoch": 0.18537168217120423, + "grad_norm": 12.866509437561035, + "kl": 1.9130859375, + "learning_rate": 9.801879589398338e-07, + "loss": 0.281, + "num_tokens": 359636288.0, + "reward": 0.99462890625, + "reward_std": 0.33961647748947144, + "rewards/accuracy_reward/mean": 0.11328125, + "rewards/accuracy_reward/std": 0.3172462284564972, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.88134765625, + "rewards/tag_count_reward/std": 0.25162631273269653, + "step": 543 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1328125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2021.0, + "completions/mean_length": 1039.720703125, + "completions/mean_terminated_length": 885.299560546875, + "completions/min_length": 243.0, + "completions/min_terminated_length": 243.0, + "epoch": 0.18571306648459504, + "grad_norm": 27.830080032348633, + "kl": 2.115234375, + "learning_rate": 9.80030330746733e-07, + "loss": 0.2714, + "num_tokens": 360241009.0, + "reward": 1.12841796875, + "reward_std": 0.3509364724159241, + "rewards/accuracy_reward/mean": 0.2265625, + "rewards/accuracy_reward/std": 0.4190165400505066, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.90185546875, + "rewards/tag_count_reward/std": 0.23722027242183685, + "step": 544 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.169921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2019.0, + "completions/mean_length": 1075.21484375, + "completions/mean_terminated_length": 876.0799560546875, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "epoch": 0.18605445079798583, + "grad_norm": 20.0777645111084, + "kl": 1.845703125, + "learning_rate": 9.798720922042316e-07, + "loss": 0.2914, + "num_tokens": 360873951.0, + "reward": 1.0576171875, + "reward_std": 0.3573155999183655, + "rewards/accuracy_reward/mean": 0.1796875, + "rewards/accuracy_reward/std": 0.38430243730545044, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8779296875, + "rewards/tag_count_reward/std": 0.2507156431674957, + "step": 545 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.138671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2002.0, + "completions/mean_length": 1059.560546875, + "completions/mean_terminated_length": 900.424072265625, + "completions/min_length": 201.0, + "completions/min_terminated_length": 201.0, + "epoch": 0.18639583511137664, + "grad_norm": 6.073132514953613, + "kl": 1.2763671875, + "learning_rate": 9.7971324353692e-07, + "loss": 0.225, + "num_tokens": 361494558.0, + "reward": 1.1376953125, + "reward_std": 0.3568900227546692, + "rewards/accuracy_reward/mean": 0.2421875, + "rewards/accuracy_reward/std": 0.42882615327835083, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8955078125, + "rewards/tag_count_reward/std": 0.2331804633140564, + "step": 546 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1898.0, + "completions/mean_length": 1169.859375, + "completions/mean_terminated_length": 987.603759765625, + "completions/min_length": 145.0, + "completions/min_terminated_length": 145.0, + "epoch": 0.18673721942476743, + "grad_norm": 16.36454963684082, + "kl": 1.921875, + "learning_rate": 9.795537849702546e-07, + "loss": 0.3132, + "num_tokens": 362170822.0, + "reward": 0.95947265625, + "reward_std": 0.3096523880958557, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.87353515625, + "rewards/tag_count_reward/std": 0.26310616731643677, + "step": 547 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.123046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1993.0, + "completions/mean_length": 1086.716796875, + "completions/mean_terminated_length": 951.8374633789062, + "completions/min_length": 249.0, + "completions/min_terminated_length": 249.0, + "epoch": 0.18707860373815824, + "grad_norm": 6.874122619628906, + "kl": 0.923828125, + "learning_rate": 9.79393716730558e-07, + "loss": 0.1929, + "num_tokens": 362812645.0, + "reward": 1.04736328125, + "reward_std": 0.2651546001434326, + "rewards/accuracy_reward/mean": 0.142578125, + "rewards/accuracy_reward/std": 0.3499840497970581, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.90478515625, + "rewards/tag_count_reward/std": 0.21860963106155396, + "step": 548 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.16015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 1167.39453125, + "completions/mean_terminated_length": 999.465087890625, + "completions/min_length": 158.0, + "completions/min_terminated_length": 158.0, + "epoch": 0.18741998805154902, + "grad_norm": 9.17322826385498, + "kl": 1.46875, + "learning_rate": 9.792330390450179e-07, + "loss": 0.2445, + "num_tokens": 363479071.0, + "reward": 0.95166015625, + "reward_std": 0.318506121635437, + "rewards/accuracy_reward/mean": 0.07459677755832672, + "rewards/accuracy_reward/std": 0.263004869222641, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.87939453125, + "rewards/tag_count_reward/std": 0.24873501062393188, + "step": 549 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.17578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1995.0, + "completions/mean_length": 1178.5703125, + "completions/mean_terminated_length": 993.14697265625, + "completions/min_length": 201.0, + "completions/min_terminated_length": 201.0, + "epoch": 0.18776137236493984, + "grad_norm": 8.53243350982666, + "kl": 1.380859375, + "learning_rate": 9.790717521416865e-07, + "loss": 0.2552, + "num_tokens": 364156995.0, + "reward": 0.99951171875, + "reward_std": 0.3222518563270569, + "rewards/accuracy_reward/mean": 0.12890625, + "rewards/accuracy_reward/std": 0.33542385697364807, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.87060546875, + "rewards/tag_count_reward/std": 0.2612071633338928, + "step": 550 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1897.0, + "completions/mean_length": 1103.326171875, + "completions/mean_terminated_length": 968.372802734375, + "completions/min_length": 255.0, + "completions/min_terminated_length": 255.0, + "epoch": 0.18810275667833062, + "grad_norm": 10.166664123535156, + "kl": 1.447265625, + "learning_rate": 9.789098562494813e-07, + "loss": 0.2518, + "num_tokens": 364802682.0, + "reward": 0.96435546875, + "reward_std": 0.30384162068367004, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.89794921875, + "rewards/tag_count_reward/std": 0.23504318296909332, + "step": 551 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.162109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2029.0, + "completions/mean_length": 1155.5546875, + "completions/mean_terminated_length": 982.8904418945312, + "completions/min_length": 127.0, + "completions/min_terminated_length": 127.0, + "epoch": 0.18844414099172144, + "grad_norm": 12.2529296875, + "kl": 1.177734375, + "learning_rate": 9.787473515981837e-07, + "loss": 0.2535, + "num_tokens": 365464006.0, + "reward": 0.9619140625, + "reward_std": 0.3028673827648163, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8759765625, + "rewards/tag_count_reward/std": 0.25121819972991943, + "step": 552 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.14453125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1972.0, + "completions/mean_length": 1006.98046875, + "completions/mean_terminated_length": 831.1004028320312, + "completions/min_length": 171.0, + "completions/min_terminated_length": 171.0, + "epoch": 0.18878552530511222, + "grad_norm": 8.00321102142334, + "kl": 1.1435546875, + "learning_rate": 9.785842384184396e-07, + "loss": 0.2241, + "num_tokens": 366055676.0, + "reward": 1.04931640625, + "reward_std": 0.3497886657714844, + "rewards/accuracy_reward/mean": 0.150390625, + "rewards/accuracy_reward/std": 0.35780346393585205, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.89892578125, + "rewards/tag_count_reward/std": 0.23337861895561218, + "step": 553 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.19140625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2009.0, + "completions/mean_length": 1160.857421875, + "completions/mean_terminated_length": 950.8574829101562, + "completions/min_length": 167.0, + "completions/min_terminated_length": 167.0, + "epoch": 0.18912690961850304, + "grad_norm": 104.50601196289062, + "kl": 3.88671875, + "learning_rate": 9.784205169417582e-07, + "loss": 0.3926, + "num_tokens": 366729459.0, + "reward": 0.94482421875, + "reward_std": 0.3352063000202179, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.86083984375, + "rewards/tag_count_reward/std": 0.2700740694999695, + "step": 554 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.142578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1989.0, + "completions/mean_length": 1085.642578125, + "completions/mean_terminated_length": 925.6150512695312, + "completions/min_length": 146.0, + "completions/min_terminated_length": 146.0, + "epoch": 0.18946829393189382, + "grad_norm": 27.38459587097168, + "kl": 2.0146484375, + "learning_rate": 9.782561874005121e-07, + "loss": 0.2778, + "num_tokens": 367361580.0, + "reward": 0.9658203125, + "reward_std": 0.29609373211860657, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8876953125, + "rewards/tag_count_reward/std": 0.2373717874288559, + "step": 555 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.16015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2024.0, + "completions/mean_length": 1115.583984375, + "completions/mean_terminated_length": 937.7744140625, + "completions/min_length": 28.0, + "completions/min_terminated_length": 28.0, + "epoch": 0.18980967824528464, + "grad_norm": 10.092026710510254, + "kl": 1.404296875, + "learning_rate": 9.78091250027937e-07, + "loss": 0.31, + "num_tokens": 368012039.0, + "reward": 0.9296875, + "reward_std": 0.29066091775894165, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.87890625, + "rewards/tag_count_reward/std": 0.24874316155910492, + "step": 556 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.212890625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2002.0, + "completions/mean_length": 1170.2734375, + "completions/mean_terminated_length": 932.8734130859375, + "completions/min_length": 233.0, + "completions/min_terminated_length": 233.0, + "epoch": 0.19015106255867542, + "grad_norm": 9.27359390258789, + "kl": 1.453125, + "learning_rate": 9.779257050581316e-07, + "loss": 0.2855, + "num_tokens": 368685555.0, + "reward": 0.95361328125, + "reward_std": 0.3154717683792114, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.86181640625, + "rewards/tag_count_reward/std": 0.2576083242893219, + "step": 557 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 1283.447265625, + "completions/mean_terminated_length": 995.7123413085938, + "completions/min_length": 264.0, + "completions/min_terminated_length": 264.0, + "epoch": 0.19049244687206623, + "grad_norm": 14.146623611450195, + "kl": 1.919921875, + "learning_rate": 9.777595527260567e-07, + "loss": 0.3542, + "num_tokens": 369419384.0, + "reward": 0.84765625, + "reward_std": 0.32374101877212524, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8046875, + "rewards/tag_count_reward/std": 0.2987048029899597, + "step": 558 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.19921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2018.0, + "completions/mean_length": 1135.689453125, + "completions/mean_terminated_length": 908.724365234375, + "completions/min_length": 271.0, + "completions/min_terminated_length": 271.0, + "epoch": 0.19083383118545702, + "grad_norm": 10.603168487548828, + "kl": 2.337890625, + "learning_rate": 9.77592793267535e-07, + "loss": 0.4041, + "num_tokens": 370075289.0, + "reward": 0.880859375, + "reward_std": 0.2896808087825775, + "rewards/accuracy_reward/mean": 0.02734375, + "rewards/accuracy_reward/std": 0.16324250400066376, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.853515625, + "rewards/tag_count_reward/std": 0.27229785919189453, + "step": 559 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.193359375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1923.0, + "completions/mean_length": 1135.6640625, + "completions/mean_terminated_length": 916.9685668945312, + "completions/min_length": 235.0, + "completions/min_terminated_length": 235.0, + "epoch": 0.19117521549884783, + "grad_norm": 437.899169921875, + "kl": 9.3515625, + "learning_rate": 9.774254269192506e-07, + "loss": 0.6456, + "num_tokens": 370730173.0, + "reward": 0.98486328125, + "reward_std": 0.371319979429245, + "rewards/accuracy_reward/mean": 0.130859375, + "rewards/accuracy_reward/std": 0.33757632970809937, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.85400390625, + "rewards/tag_count_reward/std": 0.26643362641334534, + "step": 560 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.20703125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1990.0, + "completions/mean_length": 1163.13671875, + "completions/mean_terminated_length": 932.11328125, + "completions/min_length": 309.0, + "completions/min_terminated_length": 309.0, + "epoch": 0.19151659981223862, + "grad_norm": 67.19200134277344, + "kl": 3.98046875, + "learning_rate": 9.772574539187503e-07, + "loss": 0.4641, + "num_tokens": 371398755.0, + "reward": 0.94287109375, + "reward_std": 0.38048410415649414, + "rewards/accuracy_reward/mean": 0.1015625, + "rewards/accuracy_reward/std": 0.30236753821372986, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.84130859375, + "rewards/tag_count_reward/std": 0.28297698497772217, + "step": 561 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.271484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2022.0, + "completions/mean_length": 1219.1875, + "completions/mean_terminated_length": 910.3270874023438, + "completions/min_length": 233.0, + "completions/min_terminated_length": 233.0, + "epoch": 0.19185798412562943, + "grad_norm": 13.20149040222168, + "kl": 2.9609375, + "learning_rate": 9.770888745044405e-07, + "loss": 0.4152, + "num_tokens": 372104707.0, + "reward": 0.892578125, + "reward_std": 0.3713717460632324, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.80078125, + "rewards/tag_count_reward/std": 0.3082514703273773, + "step": 562 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.212890625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1971.0, + "completions/mean_length": 1118.677734375, + "completions/mean_terminated_length": 867.3225708007812, + "completions/min_length": 109.0, + "completions/min_terminated_length": 109.0, + "epoch": 0.19219936843902022, + "grad_norm": 40.9189453125, + "kl": 2.916015625, + "learning_rate": 9.769196889155888e-07, + "loss": 0.3503, + "num_tokens": 372752974.0, + "reward": 0.91259765625, + "reward_std": 0.2583797872066498, + "rewards/accuracy_reward/mean": 0.07459677755832672, + "rewards/accuracy_reward/std": 0.263004869222641, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.84033203125, + "rewards/tag_count_reward/std": 0.284153014421463, + "step": 563 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.279296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 1222.16015625, + "completions/mean_terminated_length": 902.1192626953125, + "completions/min_length": 160.0, + "completions/min_terminated_length": 160.0, + "epoch": 0.19254075275241103, + "grad_norm": 27.644025802612305, + "kl": 2.75390625, + "learning_rate": 9.767498973923236e-07, + "loss": 0.4167, + "num_tokens": 373465888.0, + "reward": 0.92431640625, + "reward_std": 0.3687335252761841, + "rewards/accuracy_reward/mean": 0.123046875, + "rewards/accuracy_reward/std": 0.32881227135658264, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.80126953125, + "rewards/tag_count_reward/std": 0.3087652325630188, + "step": 564 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.236328125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1968.0, + "completions/mean_length": 1258.4296875, + "completions/mean_terminated_length": 1014.0869750976562, + "completions/min_length": 300.0, + "completions/min_terminated_length": 300.0, + "epoch": 0.19288213706580182, + "grad_norm": 13.675626754760742, + "kl": 2.2109375, + "learning_rate": 9.765795001756326e-07, + "loss": 0.3625, + "num_tokens": 374184876.0, + "reward": 0.91748046875, + "reward_std": 0.359503835439682, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.82568359375, + "rewards/tag_count_reward/std": 0.28755927085876465, + "step": 565 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.17578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 1161.28125, + "completions/mean_terminated_length": 972.170654296875, + "completions/min_length": 176.0, + "completions/min_terminated_length": 176.0, + "epoch": 0.19322352137919263, + "grad_norm": 13.544975280761719, + "kl": 1.93359375, + "learning_rate": 9.764084975073635e-07, + "loss": 0.3726, + "num_tokens": 374861788.0, + "reward": 0.9453125, + "reward_std": 0.3162229359149933, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.865234375, + "rewards/tag_count_reward/std": 0.263393372297287, + "step": 566 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2024.0, + "completions/mean_length": 1153.486328125, + "completions/mean_terminated_length": 903.0224609375, + "completions/min_length": 127.0, + "completions/min_terminated_length": 127.0, + "epoch": 0.19356490569258342, + "grad_norm": 16.100940704345703, + "kl": 1.78125, + "learning_rate": 9.762368896302234e-07, + "loss": 0.4092, + "num_tokens": 375536773.0, + "reward": 0.90087890625, + "reward_std": 0.3287081718444824, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.83447265625, + "rewards/tag_count_reward/std": 0.2876589298248291, + "step": 567 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.201171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 1151.13671875, + "completions/mean_terminated_length": 925.2763061523438, + "completions/min_length": 140.0, + "completions/min_terminated_length": 140.0, + "epoch": 0.19390629000597423, + "grad_norm": 108.90221405029297, + "kl": 4.123046875, + "learning_rate": 9.760646767877784e-07, + "loss": 0.3934, + "num_tokens": 376199451.0, + "reward": 0.9091796875, + "reward_std": 0.3188011348247528, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8447265625, + "rewards/tag_count_reward/std": 0.28336378931999207, + "step": 568 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2265625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2020.0, + "completions/mean_length": 1247.072265625, + "completions/mean_terminated_length": 1012.4570922851562, + "completions/min_length": 198.0, + "completions/min_terminated_length": 198.0, + "epoch": 0.19424767431936502, + "grad_norm": 6.915520668029785, + "kl": 1.462890625, + "learning_rate": 9.758918592244528e-07, + "loss": 0.2869, + "num_tokens": 376910864.0, + "reward": 0.9052734375, + "reward_std": 0.31668218970298767, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8212890625, + "rewards/tag_count_reward/std": 0.29724088311195374, + "step": 569 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.244140625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1875.0, + "completions/mean_length": 1207.375, + "completions/mean_terminated_length": 935.8553466796875, + "completions/min_length": 295.0, + "completions/min_terminated_length": 295.0, + "epoch": 0.19458905863275583, + "grad_norm": 6.648573875427246, + "kl": 1.451171875, + "learning_rate": 9.757184371855298e-07, + "loss": 0.3194, + "num_tokens": 377602336.0, + "reward": 0.89208984375, + "reward_std": 0.34159648418426514, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.80615234375, + "rewards/tag_count_reward/std": 0.3091023564338684, + "step": 570 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.236328125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 1252.3671875, + "completions/mean_terminated_length": 1006.1483154296875, + "completions/min_length": 255.0, + "completions/min_terminated_length": 255.0, + "epoch": 0.19493044294614661, + "grad_norm": 5.759603500366211, + "kl": 1.298828125, + "learning_rate": 9.7554441091715e-07, + "loss": 0.2955, + "num_tokens": 378328108.0, + "reward": 0.8935546875, + "reward_std": 0.35490304231643677, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8173828125, + "rewards/tag_count_reward/std": 0.303035169839859, + "step": 571 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1947.0, + "completions/mean_length": 1162.66796875, + "completions/mean_terminated_length": 947.7815551757812, + "completions/min_length": 269.0, + "completions/min_terminated_length": 269.0, + "epoch": 0.19527182725953743, + "grad_norm": 12.158112525939941, + "kl": 1.578125, + "learning_rate": 9.753697806663124e-07, + "loss": 0.2999, + "num_tokens": 379000018.0, + "reward": 0.9765625, + "reward_std": 0.3368375897407532, + "rewards/accuracy_reward/mean": 0.12890625, + "rewards/accuracy_reward/std": 0.33542385697364807, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.84765625, + "rewards/tag_count_reward/std": 0.2810630798339844, + "step": 572 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1993.0, + "completions/mean_length": 1231.03515625, + "completions/mean_terminated_length": 1012.6386108398438, + "completions/min_length": 233.0, + "completions/min_terminated_length": 233.0, + "epoch": 0.19561321157292821, + "grad_norm": 4.52598762512207, + "kl": 1.0849609375, + "learning_rate": 9.75194546680872e-07, + "loss": 0.2984, + "num_tokens": 379706836.0, + "reward": 0.88427734375, + "reward_std": 0.2944050431251526, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.84130859375, + "rewards/tag_count_reward/std": 0.2821112275123596, + "step": 573 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.27734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2011.0, + "completions/mean_length": 1284.548828125, + "completions/mean_terminated_length": 991.5486450195312, + "completions/min_length": 152.0, + "completions/min_terminated_length": 152.0, + "epoch": 0.19595459588631903, + "grad_norm": 5.709986686706543, + "kl": 1.51953125, + "learning_rate": 9.750187092095422e-07, + "loss": 0.3602, + "num_tokens": 380441357.0, + "reward": 0.84228515625, + "reward_std": 0.31777364015579224, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.80322265625, + "rewards/tag_count_reward/std": 0.3060453534126282, + "step": 574 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.23046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2004.0, + "completions/mean_length": 1172.1015625, + "completions/mean_terminated_length": 909.776611328125, + "completions/min_length": 217.0, + "completions/min_terminated_length": 217.0, + "epoch": 0.1962959801997098, + "grad_norm": 18.69489860534668, + "kl": 1.7578125, + "learning_rate": 9.748422685018911e-07, + "loss": 0.3303, + "num_tokens": 381115553.0, + "reward": 1.0009765625, + "reward_std": 0.39446723461151123, + "rewards/accuracy_reward/mean": 0.15625, + "rewards/accuracy_reward/std": 0.36344730854034424, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8447265625, + "rewards/tag_count_reward/std": 0.28032606840133667, + "step": 575 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2421875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 1263.09765625, + "completions/mean_terminated_length": 1012.2525634765625, + "completions/min_length": 228.0, + "completions/min_terminated_length": 228.0, + "epoch": 0.19663736451310063, + "grad_norm": 17.96137046813965, + "kl": 1.458984375, + "learning_rate": 9.74665224808345e-07, + "loss": 0.29, + "num_tokens": 381847219.0, + "reward": 0.8447265625, + "reward_std": 0.28833621740341187, + "rewards/accuracy_reward/mean": 0.02822580561041832, + "rewards/accuracy_reward/std": 0.1657845675945282, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8173828125, + "rewards/tag_count_reward/std": 0.2989718019962311, + "step": 576 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2014.0, + "completions/mean_length": 1207.884765625, + "completions/mean_terminated_length": 983.2994995117188, + "completions/min_length": 277.0, + "completions/min_terminated_length": 277.0, + "epoch": 0.1969787488264914, + "grad_norm": 6.886361598968506, + "kl": 1.271484375, + "learning_rate": 9.744875783801844e-07, + "loss": 0.2947, + "num_tokens": 382545448.0, + "reward": 0.9453125, + "reward_std": 0.34676775336265564, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.84765625, + "rewards/tag_count_reward/std": 0.27843984961509705, + "step": 577 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2028.0, + "completions/mean_length": 1181.689453125, + "completions/mean_terminated_length": 1001.88916015625, + "completions/min_length": 216.0, + "completions/min_terminated_length": 216.0, + "epoch": 0.19732013313988223, + "grad_norm": 10.539137840270996, + "kl": 1.26171875, + "learning_rate": 9.743093294695461e-07, + "loss": 0.2806, + "num_tokens": 383226905.0, + "reward": 0.94677734375, + "reward_std": 0.33402007818222046, + "rewards/accuracy_reward/mean": 0.08064515888690948, + "rewards/accuracy_reward/std": 0.2725643217563629, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.86865234375, + "rewards/tag_count_reward/std": 0.2667275369167328, + "step": 578 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.173828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1972.0, + "completions/mean_length": 1234.08203125, + "completions/mean_terminated_length": 1062.8321533203125, + "completions/min_length": 238.0, + "completions/min_terminated_length": 238.0, + "epoch": 0.197661517453273, + "grad_norm": 6.447632312774658, + "kl": 0.826171875, + "learning_rate": 9.741304783294218e-07, + "loss": 0.2191, + "num_tokens": 383938131.0, + "reward": 0.96484375, + "reward_std": 0.272844135761261, + "rewards/accuracy_reward/mean": 0.087890625, + "rewards/accuracy_reward/std": 0.2834126651287079, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.876953125, + "rewards/tag_count_reward/std": 0.2507251501083374, + "step": 579 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 1177.27734375, + "completions/mean_terminated_length": 1052.888427734375, + "completions/min_length": 271.0, + "completions/min_terminated_length": 271.0, + "epoch": 0.19800290176666382, + "grad_norm": 8.025528907775879, + "kl": 0.7685546875, + "learning_rate": 9.739510252136584e-07, + "loss": 0.2053, + "num_tokens": 384619873.0, + "reward": 1.0771484375, + "reward_std": 0.3427858352661133, + "rewards/accuracy_reward/mean": 0.16796875, + "rewards/accuracy_reward/std": 0.374204158782959, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9091796875, + "rewards/tag_count_reward/std": 0.21964147686958313, + "step": 580 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 1070.193359375, + "completions/mean_terminated_length": 915.337158203125, + "completions/min_length": 228.0, + "completions/min_terminated_length": 228.0, + "epoch": 0.1983442860800546, + "grad_norm": 7.305599212646484, + "kl": 0.5322265625, + "learning_rate": 9.737709703769562e-07, + "loss": 0.1573, + "num_tokens": 385249684.0, + "reward": 0.982421875, + "reward_std": 0.26694631576538086, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.90234375, + "rewards/tag_count_reward/std": 0.23194991052150726, + "step": 581 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.13671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 1132.24609375, + "completions/mean_terminated_length": 987.2172241210938, + "completions/min_length": 271.0, + "completions/min_terminated_length": 271.0, + "epoch": 0.19868567039344542, + "grad_norm": 8.776261329650879, + "kl": 0.431640625, + "learning_rate": 9.735903140748702e-07, + "loss": 0.2056, + "num_tokens": 385915458.0, + "reward": 1.03662109375, + "reward_std": 0.29240643978118896, + "rewards/accuracy_reward/mean": 0.1328125, + "rewards/accuracy_reward/std": 0.33970388770103455, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.90380859375, + "rewards/tag_count_reward/std": 0.22535066306591034, + "step": 582 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.099609375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 1057.48046875, + "completions/mean_terminated_length": 947.9002075195312, + "completions/min_length": 235.0, + "completions/min_terminated_length": 235.0, + "epoch": 0.1990270547068362, + "grad_norm": 8.846368789672852, + "kl": 0.49609375, + "learning_rate": 9.734090565638092e-07, + "loss": 0.1636, + "num_tokens": 386528280.0, + "reward": 1.041015625, + "reward_std": 0.28835421800613403, + "rewards/accuracy_reward/mean": 0.11328125, + "rewards/accuracy_reward/std": 0.3172462284564972, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.927734375, + "rewards/tag_count_reward/std": 0.1969740390777588, + "step": 583 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.130859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 1148.72265625, + "completions/mean_terminated_length": 1013.3258666992188, + "completions/min_length": 336.0, + "completions/min_terminated_length": 336.0, + "epoch": 0.19936843902022702, + "grad_norm": 3.6344919204711914, + "kl": 0.74609375, + "learning_rate": 9.73227198101035e-07, + "loss": 0.2193, + "num_tokens": 387198378.0, + "reward": 0.97216796875, + "reward_std": 0.25956839323043823, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.90576171875, + "rewards/tag_count_reward/std": 0.2195909023284912, + "step": 584 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.11328125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2018.0, + "completions/mean_length": 989.21875, + "completions/mean_terminated_length": 853.9559326171875, + "completions/min_length": 297.0, + "completions/min_terminated_length": 297.0, + "epoch": 0.1997098233336178, + "grad_norm": 7.630433082580566, + "kl": 0.58349609375, + "learning_rate": 9.730447389446623e-07, + "loss": 0.1664, + "num_tokens": 387774170.0, + "reward": 1.05322265625, + "reward_std": 0.28049057722091675, + "rewards/accuracy_reward/mean": 0.12890625, + "rewards/accuracy_reward/std": 0.33542385697364807, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.92431640625, + "rewards/tag_count_reward/std": 0.19157785177230835, + "step": 585 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 1135.650390625, + "completions/mean_terminated_length": 956.591064453125, + "completions/min_length": 211.0, + "completions/min_terminated_length": 211.0, + "epoch": 0.20005120764700862, + "grad_norm": 43.59455871582031, + "kl": 1.6923828125, + "learning_rate": 9.728616793536587e-07, + "loss": 0.2915, + "num_tokens": 388433607.0, + "reward": 0.939453125, + "reward_std": 0.3105131983757019, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.873046875, + "rewards/tag_count_reward/std": 0.26217159628868103, + "step": 586 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1983.0, + "completions/mean_length": 1157.76953125, + "completions/mean_terminated_length": 952.331787109375, + "completions/min_length": 259.0, + "completions/min_terminated_length": 259.0, + "epoch": 0.2003925919603994, + "grad_norm": 2.1202924251556396, + "kl": 0.60986328125, + "learning_rate": 9.726780195878438e-07, + "loss": 0.2669, + "num_tokens": 389108625.0, + "reward": 0.93701171875, + "reward_std": 0.3265402317047119, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.85498046875, + "rewards/tag_count_reward/std": 0.2777453064918518, + "step": 587 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2023.0, + "completions/mean_length": 1094.56640625, + "completions/mean_terminated_length": 918.004638671875, + "completions/min_length": 195.0, + "completions/min_terminated_length": 195.0, + "epoch": 0.20073397627379022, + "grad_norm": 6.691629409790039, + "kl": 0.939453125, + "learning_rate": 9.724937599078888e-07, + "loss": 0.2613, + "num_tokens": 389744771.0, + "reward": 0.97021484375, + "reward_std": 0.32121267914772034, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.87255859375, + "rewards/tag_count_reward/std": 0.2663331925868988, + "step": 588 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.208984375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2003.0, + "completions/mean_length": 1166.4765625, + "completions/mean_terminated_length": 933.5802612304688, + "completions/min_length": 211.0, + "completions/min_terminated_length": 211.0, + "epoch": 0.201075360587181, + "grad_norm": 6.024787902832031, + "kl": 0.8759765625, + "learning_rate": 9.72308900575317e-07, + "loss": 0.2851, + "num_tokens": 390420055.0, + "reward": 0.9619140625, + "reward_std": 0.3914545178413391, + "rewards/accuracy_reward/mean": 0.125, + "rewards/accuracy_reward/std": 0.3310528099536896, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8408203125, + "rewards/tag_count_reward/std": 0.2897527813911438, + "step": 589 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.142578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1959.0, + "completions/mean_length": 1046.951171875, + "completions/mean_terminated_length": 880.4898071289062, + "completions/min_length": 232.0, + "completions/min_terminated_length": 232.0, + "epoch": 0.20141674490057182, + "grad_norm": 4.769101142883301, + "kl": 0.75390625, + "learning_rate": 9.72123441852502e-07, + "loss": 0.2474, + "num_tokens": 391023566.0, + "reward": 1.03515625, + "reward_std": 0.34677445888519287, + "rewards/accuracy_reward/mean": 0.150390625, + "rewards/accuracy_reward/std": 0.35780346393585205, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.884765625, + "rewards/tag_count_reward/std": 0.2544175982475281, + "step": 590 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.15234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2022.0, + "completions/mean_length": 1102.0078125, + "completions/mean_terminated_length": 931.9907836914062, + "completions/min_length": 275.0, + "completions/min_terminated_length": 275.0, + "epoch": 0.2017581292139626, + "grad_norm": 2.9391894340515137, + "kl": 0.791015625, + "learning_rate": 9.719373840026686e-07, + "loss": 0.2679, + "num_tokens": 391669458.0, + "reward": 0.978515625, + "reward_std": 0.29991257190704346, + "rewards/accuracy_reward/mean": 0.099609375, + "rewards/accuracy_reward/std": 0.29977133870124817, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.87890625, + "rewards/tag_count_reward/std": 0.2626158595085144, + "step": 591 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.123046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2025.0, + "completions/mean_length": 1028.095703125, + "completions/mean_terminated_length": 884.9910888671875, + "completions/min_length": 113.0, + "completions/min_terminated_length": 113.0, + "epoch": 0.20209951352735342, + "grad_norm": 1.3754154443740845, + "kl": 0.55029296875, + "learning_rate": 9.717507272898922e-07, + "loss": 0.2428, + "num_tokens": 392269523.0, + "reward": 1.03662109375, + "reward_std": 0.3236616253852844, + "rewards/accuracy_reward/mean": 0.13671875, + "rewards/accuracy_reward/std": 0.3438861668109894, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.89990234375, + "rewards/tag_count_reward/std": 0.24050459265708923, + "step": 592 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2006.0, + "completions/mean_length": 1076.001953125, + "completions/mean_terminated_length": 942.082275390625, + "completions/min_length": 124.0, + "completions/min_terminated_length": 124.0, + "epoch": 0.2024408978407442, + "grad_norm": 3.039691686630249, + "kl": 0.48388671875, + "learning_rate": 9.715634719790978e-07, + "loss": 0.1811, + "num_tokens": 392902180.0, + "reward": 0.990234375, + "reward_std": 0.2621752619743347, + "rewards/accuracy_reward/mean": 0.08669354766607285, + "rewards/accuracy_reward/std": 0.281669557094574, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.90625, + "rewards/tag_count_reward/std": 0.23198285698890686, + "step": 593 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 975.173828125, + "completions/mean_terminated_length": 832.7632446289062, + "completions/min_length": 175.0, + "completions/min_terminated_length": 175.0, + "epoch": 0.20278228215413502, + "grad_norm": 1.9965825080871582, + "kl": 0.48779296875, + "learning_rate": 9.713756183360597e-07, + "loss": 0.2001, + "num_tokens": 393472061.0, + "reward": 1.07666015625, + "reward_std": 0.3201131820678711, + "rewards/accuracy_reward/mean": 0.169921875, + "rewards/accuracy_reward/std": 0.3759314715862274, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.90673828125, + "rewards/tag_count_reward/std": 0.22979721426963806, + "step": 594 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.158203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2031.0, + "completions/mean_length": 1195.353515625, + "completions/mean_terminated_length": 1035.111328125, + "completions/min_length": 275.0, + "completions/min_terminated_length": 275.0, + "epoch": 0.2031236664675258, + "grad_norm": 1.442400574684143, + "kl": 0.55322265625, + "learning_rate": 9.711871666274021e-07, + "loss": 0.2017, + "num_tokens": 394158898.0, + "reward": 0.9970703125, + "reward_std": 0.3248283267021179, + "rewards/accuracy_reward/mean": 0.11328125, + "rewards/accuracy_reward/std": 0.3172462284564972, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8837890625, + "rewards/tag_count_reward/std": 0.2554128170013428, + "step": 595 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.107421875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 1080.25390625, + "completions/mean_terminated_length": 963.7855834960938, + "completions/min_length": 147.0, + "completions/min_terminated_length": 147.0, + "epoch": 0.20346505078091662, + "grad_norm": 1.8731517791748047, + "kl": 0.53271484375, + "learning_rate": 9.709981171205977e-07, + "loss": 0.1676, + "num_tokens": 394777940.0, + "reward": 1.068359375, + "reward_std": 0.30113649368286133, + "rewards/accuracy_reward/mean": 0.16532258689403534, + "rewards/accuracy_reward/std": 0.371846467256546, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.908203125, + "rewards/tag_count_reward/std": 0.21979151666164398, + "step": 596 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.18359375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1987.0, + "completions/mean_length": 1195.24609375, + "completions/mean_terminated_length": 1003.4784545898438, + "completions/min_length": 158.0, + "completions/min_terminated_length": 158.0, + "epoch": 0.2038064350943074, + "grad_norm": 3.4311375617980957, + "kl": 0.9697265625, + "learning_rate": 9.708084700839678e-07, + "loss": 0.2665, + "num_tokens": 395461810.0, + "reward": 0.95458984375, + "reward_std": 0.36056941747665405, + "rewards/accuracy_reward/mean": 0.10080645233392715, + "rewards/accuracy_reward/std": 0.30137622356414795, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.85693359375, + "rewards/tag_count_reward/std": 0.2796345353126526, + "step": 597 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.119140625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 1077.2734375, + "completions/mean_terminated_length": 945.9778442382812, + "completions/min_length": 296.0, + "completions/min_terminated_length": 296.0, + "epoch": 0.20414781940769822, + "grad_norm": 3.1357645988464355, + "kl": 0.65234375, + "learning_rate": 9.706182257866812e-07, + "loss": 0.1647, + "num_tokens": 396095806.0, + "reward": 1.0810546875, + "reward_std": 0.2692202031612396, + "rewards/accuracy_reward/mean": 0.169921875, + "rewards/accuracy_reward/std": 0.3759314715862274, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9111328125, + "rewards/tag_count_reward/std": 0.2280760258436203, + "step": 598 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.142578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1994.0, + "completions/mean_length": 1061.494140625, + "completions/mean_terminated_length": 897.4510498046875, + "completions/min_length": 72.0, + "completions/min_terminated_length": 72.0, + "epoch": 0.204489203721089, + "grad_norm": 3.2705795764923096, + "kl": 0.9462890625, + "learning_rate": 9.704273844987555e-07, + "loss": 0.2939, + "num_tokens": 396724923.0, + "reward": 1.00537109375, + "reward_std": 0.3218349814414978, + "rewards/accuracy_reward/mean": 0.1171875, + "rewards/accuracy_reward/std": 0.32195815443992615, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.88818359375, + "rewards/tag_count_reward/std": 0.24445319175720215, + "step": 599 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.115234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2012.0, + "completions/mean_length": 1054.888671875, + "completions/mean_terminated_length": 925.5430297851562, + "completions/min_length": 203.0, + "completions/min_terminated_length": 203.0, + "epoch": 0.20483058803447982, + "grad_norm": 10.378336906433105, + "kl": 1.1484375, + "learning_rate": 9.702359464910546e-07, + "loss": 0.2279, + "num_tokens": 397344994.0, + "reward": 1.06201171875, + "reward_std": 0.2892497181892395, + "rewards/accuracy_reward/mean": 0.158203125, + "rewards/accuracy_reward/std": 0.36528825759887695, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.90380859375, + "rewards/tag_count_reward/std": 0.23124386370182037, + "step": 600 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.103515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1989.0, + "completions/mean_length": 1068.806640625, + "completions/mean_terminated_length": 955.74072265625, + "completions/min_length": 261.0, + "completions/min_terminated_length": 261.0, + "epoch": 0.20517197234787063, + "grad_norm": 6.7024993896484375, + "kl": 1.14453125, + "learning_rate": 9.700439120352898e-07, + "loss": 0.2144, + "num_tokens": 397968351.0, + "reward": 1.05126953125, + "reward_std": 0.3121991753578186, + "rewards/accuracy_reward/mean": 0.1328125, + "rewards/accuracy_reward/std": 0.33970388770103455, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.91845703125, + "rewards/tag_count_reward/std": 0.20940732955932617, + "step": 601 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.095703125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2008.0, + "completions/mean_length": 1000.607421875, + "completions/mean_terminated_length": 889.76025390625, + "completions/min_length": 236.0, + "completions/min_terminated_length": 236.0, + "epoch": 0.20551335666126141, + "grad_norm": 6.8327250480651855, + "kl": 0.7802734375, + "learning_rate": 9.69851281404019e-07, + "loss": 0.1916, + "num_tokens": 398559590.0, + "reward": 1.0224609375, + "reward_std": 0.22936491668224335, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9248046875, + "rewards/tag_count_reward/std": 0.20141369104385376, + "step": 602 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.119140625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 1099.17578125, + "completions/mean_terminated_length": 970.8425903320312, + "completions/min_length": 271.0, + "completions/min_terminated_length": 271.0, + "epoch": 0.20585474097465223, + "grad_norm": 2.3790547847747803, + "kl": 0.669921875, + "learning_rate": 9.696580548706462e-07, + "loss": 0.1783, + "num_tokens": 399197216.0, + "reward": 1.03662109375, + "reward_std": 0.28269410133361816, + "rewards/accuracy_reward/mean": 0.119140625, + "rewards/accuracy_reward/std": 0.32427072525024414, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.91748046875, + "rewards/tag_count_reward/std": 0.21077179908752441, + "step": 603 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.099609375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2016.0, + "completions/mean_length": 1123.58984375, + "completions/mean_terminated_length": 1021.3232421875, + "completions/min_length": 262.0, + "completions/min_terminated_length": 262.0, + "epoch": 0.20619612528804301, + "grad_norm": 5.899009704589844, + "kl": 0.904296875, + "learning_rate": 9.69464232709421e-07, + "loss": 0.1753, + "num_tokens": 399851470.0, + "reward": 1.0283203125, + "reward_std": 0.2769412398338318, + "rewards/accuracy_reward/mean": 0.109375, + "rewards/accuracy_reward/std": 0.31241437792778015, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9189453125, + "rewards/tag_count_reward/std": 0.2063627392053604, + "step": 604 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 1084.873046875, + "completions/mean_terminated_length": 985.2391967773438, + "completions/min_length": 161.0, + "completions/min_terminated_length": 161.0, + "epoch": 0.20653750960143383, + "grad_norm": 1.9711962938308716, + "kl": 0.537109375, + "learning_rate": 9.692698151954383e-07, + "loss": 0.1471, + "num_tokens": 400480045.0, + "reward": 1.08837890625, + "reward_std": 0.2671680748462677, + "rewards/accuracy_reward/mean": 0.15625, + "rewards/accuracy_reward/std": 0.36344730854034424, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.93212890625, + "rewards/tag_count_reward/std": 0.18214841187000275, + "step": 605 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.068359375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 1009.521484375, + "completions/mean_terminated_length": 933.3228149414062, + "completions/min_length": 281.0, + "completions/min_terminated_length": 281.0, + "epoch": 0.2068788939148246, + "grad_norm": 4.671616077423096, + "kl": 0.58447265625, + "learning_rate": 9.690748026046386e-07, + "loss": 0.1619, + "num_tokens": 401074696.0, + "reward": 1.04296875, + "reward_std": 0.2465844303369522, + "rewards/accuracy_reward/mean": 0.1015625, + "rewards/accuracy_reward/std": 0.30236753821372986, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.94140625, + "rewards/tag_count_reward/std": 0.17128607630729675, + "step": 606 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1923.0, + "completions/mean_length": 1083.345703125, + "completions/mean_terminated_length": 997.1425170898438, + "completions/min_length": 280.0, + "completions/min_terminated_length": 280.0, + "epoch": 0.20722027822821543, + "grad_norm": 2.694904327392578, + "kl": 0.59326171875, + "learning_rate": 9.688791952138068e-07, + "loss": 0.1555, + "num_tokens": 401703593.0, + "reward": 1.0068359375, + "reward_std": 0.21862350404262543, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9345703125, + "rewards/tag_count_reward/std": 0.18798606097698212, + "step": 607 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 1075.267578125, + "completions/mean_terminated_length": 988.342529296875, + "completions/min_length": 233.0, + "completions/min_terminated_length": 233.0, + "epoch": 0.2075616625416062, + "grad_norm": 2.220994710922241, + "kl": 0.68994140625, + "learning_rate": 9.686829933005709e-07, + "loss": 0.1197, + "num_tokens": 402333810.0, + "reward": 1.02587890625, + "reward_std": 0.21877221763134003, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.93603515625, + "rewards/tag_count_reward/std": 0.18222182989120483, + "step": 608 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08984375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1978.0, + "completions/mean_length": 1006.33984375, + "completions/mean_terminated_length": 903.5150146484375, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "epoch": 0.20790304685499703, + "grad_norm": 1.1816500425338745, + "kl": 0.6005859375, + "learning_rate": 9.684861971434043e-07, + "loss": 0.1771, + "num_tokens": 402925808.0, + "reward": 0.99365234375, + "reward_std": 0.27654939889907837, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.91748046875, + "rewards/tag_count_reward/std": 0.21019071340560913, + "step": 609 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.091796875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1991.0, + "completions/mean_length": 1075.279296875, + "completions/mean_terminated_length": 976.9613037109375, + "completions/min_length": 224.0, + "completions/min_terminated_length": 224.0, + "epoch": 0.2082444311683878, + "grad_norm": 1.2841227054595947, + "kl": 0.580078125, + "learning_rate": 9.682888070216231e-07, + "loss": 0.1514, + "num_tokens": 403563439.0, + "reward": 1.0458984375, + "reward_std": 0.30254921317100525, + "rewards/accuracy_reward/mean": 0.125, + "rewards/accuracy_reward/std": 0.3310423493385315, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9208984375, + "rewards/tag_count_reward/std": 0.20354676246643066, + "step": 610 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1889.0, + "completions/mean_length": 1040.328125, + "completions/mean_terminated_length": 931.272705078125, + "completions/min_length": 288.0, + "completions/min_terminated_length": 288.0, + "epoch": 0.20858581548177862, + "grad_norm": 2.6428682804107666, + "kl": 0.8291015625, + "learning_rate": 9.680908232153865e-07, + "loss": 0.2066, + "num_tokens": 404175207.0, + "reward": 1.033203125, + "reward_std": 0.2689477801322937, + "rewards/accuracy_reward/mean": 0.11328125, + "rewards/accuracy_reward/std": 0.3172462284564972, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.919921875, + "rewards/tag_count_reward/std": 0.2037649303674698, + "step": 611 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 1054.154296875, + "completions/mean_terminated_length": 932.1030883789062, + "completions/min_length": 207.0, + "completions/min_terminated_length": 207.0, + "epoch": 0.2089271997951694, + "grad_norm": 3.411529064178467, + "kl": 0.8994140625, + "learning_rate": 9.67892246005696e-07, + "loss": 0.2368, + "num_tokens": 404799126.0, + "reward": 0.93115234375, + "reward_std": 0.2242891639471054, + "rewards/accuracy_reward/mean": 0.01953125, + "rewards/accuracy_reward/std": 0.1385180652141571, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.91162109375, + "rewards/tag_count_reward/std": 0.22201864421367645, + "step": 612 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.076171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2021.0, + "completions/mean_length": 1026.0078125, + "completions/mean_terminated_length": 941.7420654296875, + "completions/min_length": 147.0, + "completions/min_terminated_length": 147.0, + "epoch": 0.20926858410856022, + "grad_norm": 2.8635215759277344, + "kl": 0.857421875, + "learning_rate": 9.67693075674396e-07, + "loss": 0.1983, + "num_tokens": 405396170.0, + "reward": 1.04638671875, + "reward_std": 0.26304829120635986, + "rewards/accuracy_reward/mean": 0.109375, + "rewards/accuracy_reward/std": 0.31241437792778015, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.93701171875, + "rewards/tag_count_reward/std": 0.18323110044002533, + "step": 613 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 983.9609375, + "completions/mean_terminated_length": 908.276123046875, + "completions/min_length": 121.0, + "completions/min_terminated_length": 121.0, + "epoch": 0.209609968421951, + "grad_norm": 1.214871883392334, + "kl": 0.53515625, + "learning_rate": 9.674933125041722e-07, + "loss": 0.1449, + "num_tokens": 405978086.0, + "reward": 1.05126953125, + "reward_std": 0.2484574317932129, + "rewards/accuracy_reward/mean": 0.1171875, + "rewards/accuracy_reward/std": 0.32195815443992615, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.93408203125, + "rewards/tag_count_reward/std": 0.17949029803276062, + "step": 614 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08984375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1961.0, + "completions/mean_length": 1032.369140625, + "completions/mean_terminated_length": 932.1137084960938, + "completions/min_length": 181.0, + "completions/min_terminated_length": 181.0, + "epoch": 0.20995135273534182, + "grad_norm": 5.269324779510498, + "kl": 0.71533203125, + "learning_rate": 9.672929567785517e-07, + "loss": 0.2549, + "num_tokens": 406588067.0, + "reward": 1.02783203125, + "reward_std": 0.2760624885559082, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.92236328125, + "rewards/tag_count_reward/std": 0.19585449993610382, + "step": 615 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.095703125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1986.0, + "completions/mean_length": 978.466796875, + "completions/mean_terminated_length": 865.2764282226562, + "completions/min_length": 219.0, + "completions/min_terminated_length": 219.0, + "epoch": 0.2102927370487326, + "grad_norm": 5.354698181152344, + "kl": 0.953125, + "learning_rate": 9.67092008781903e-07, + "loss": 0.2332, + "num_tokens": 407172866.0, + "reward": 0.99609375, + "reward_std": 0.24358966946601868, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.919921875, + "rewards/tag_count_reward/std": 0.20074127614498138, + "step": 616 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.123046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 1043.744140625, + "completions/mean_terminated_length": 902.835205078125, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "epoch": 0.21063412136212342, + "grad_norm": 2.5083742141723633, + "kl": 1.2353515625, + "learning_rate": 9.668904687994351e-07, + "loss": 0.2444, + "num_tokens": 407790127.0, + "reward": 0.96826171875, + "reward_std": 0.27520516514778137, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.89794921875, + "rewards/tag_count_reward/std": 0.22330179810523987, + "step": 617 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.1171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1990.0, + "completions/mean_length": 1018.123046875, + "completions/mean_terminated_length": 881.4136962890625, + "completions/min_length": 233.0, + "completions/min_terminated_length": 233.0, + "epoch": 0.2109755056755142, + "grad_norm": 2.2218728065490723, + "kl": 1.0400390625, + "learning_rate": 9.66688337117197e-07, + "loss": 0.2165, + "num_tokens": 408380542.0, + "reward": 1.08837890625, + "reward_std": 0.2701031565666199, + "rewards/accuracy_reward/mean": 0.17578125, + "rewards/accuracy_reward/std": 0.3810062110424042, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.91259765625, + "rewards/tag_count_reward/std": 0.20284785330295563, + "step": 618 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2017.0, + "completions/mean_length": 1019.171875, + "completions/mean_terminated_length": 872.1964721679688, + "completions/min_length": 279.0, + "completions/min_terminated_length": 279.0, + "epoch": 0.21131688998890502, + "grad_norm": 4.6177239418029785, + "kl": 1.50390625, + "learning_rate": 9.664856140220778e-07, + "loss": 0.3002, + "num_tokens": 408980726.0, + "reward": 0.96923828125, + "reward_std": 0.25260913372039795, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.88916015625, + "rewards/tag_count_reward/std": 0.22293777763843536, + "step": 619 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.134765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1983.0, + "completions/mean_length": 944.771484375, + "completions/mean_terminated_length": 772.9368286132812, + "completions/min_length": 120.0, + "completions/min_terminated_length": 120.0, + "epoch": 0.2116582743022958, + "grad_norm": 3.555220127105713, + "kl": 1.1787109375, + "learning_rate": 9.662822998018056e-07, + "loss": 0.295, + "num_tokens": 409541921.0, + "reward": 0.9599609375, + "reward_std": 0.29147058725357056, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8876953125, + "rewards/tag_count_reward/std": 0.23478132486343384, + "step": 620 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.177734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 1059.083984375, + "completions/mean_terminated_length": 845.3278198242188, + "completions/min_length": 211.0, + "completions/min_terminated_length": 211.0, + "epoch": 0.21199965861568662, + "grad_norm": 3.064396619796753, + "kl": 1.318359375, + "learning_rate": 9.66078394744948e-07, + "loss": 0.339, + "num_tokens": 410162044.0, + "reward": 0.9443359375, + "reward_std": 0.30136334896087646, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8662109375, + "rewards/tag_count_reward/std": 0.24364906549453735, + "step": 621 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.095703125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1915.0, + "completions/mean_length": 923.7109375, + "completions/mean_terminated_length": 804.7257080078125, + "completions/min_length": 266.0, + "completions/min_terminated_length": 266.0, + "epoch": 0.2123410429290774, + "grad_norm": 2.5860583782196045, + "kl": 1.013671875, + "learning_rate": 9.65873899140911e-07, + "loss": 0.2859, + "num_tokens": 410713128.0, + "reward": 1.04443359375, + "reward_std": 0.29433169960975647, + "rewards/accuracy_reward/mean": 0.138671875, + "rewards/accuracy_reward/std": 0.34594178199768066, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.90576171875, + "rewards/tag_count_reward/std": 0.20519430935382843, + "step": 622 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.099609375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2007.0, + "completions/mean_length": 875.408203125, + "completions/mean_terminated_length": 745.6854858398438, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "epoch": 0.21268242724246822, + "grad_norm": 2.3693439960479736, + "kl": 1.138671875, + "learning_rate": 9.656688132799382e-07, + "loss": 0.2828, + "num_tokens": 411235097.0, + "reward": 1.05859375, + "reward_std": 0.32291650772094727, + "rewards/accuracy_reward/mean": 0.158203125, + "rewards/accuracy_reward/std": 0.36528825759887695, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.900390625, + "rewards/tag_count_reward/std": 0.21804559230804443, + "step": 623 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.20703125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2000.0, + "completions/mean_length": 1061.150390625, + "completions/mean_terminated_length": 803.5, + "completions/min_length": 154.0, + "completions/min_terminated_length": 154.0, + "epoch": 0.213023811555859, + "grad_norm": 2.54702091217041, + "kl": 1.3125, + "learning_rate": 9.65463137453112e-07, + "loss": 0.3377, + "num_tokens": 411867830.0, + "reward": 0.92919921875, + "reward_std": 0.3174704909324646, + "rewards/accuracy_reward/mean": 0.0927419364452362, + "rewards/accuracy_reward/std": 0.2903633117675781, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.83935546875, + "rewards/tag_count_reward/std": 0.2597397565841675, + "step": 624 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.20703125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 1057.361328125, + "completions/mean_terminated_length": 798.7216796875, + "completions/min_length": 192.0, + "completions/min_terminated_length": 192.0, + "epoch": 0.21336519586924982, + "grad_norm": 11.175084114074707, + "kl": 1.546875, + "learning_rate": 9.652568719523516e-07, + "loss": 0.352, + "num_tokens": 412487919.0, + "reward": 0.921875, + "reward_std": 0.27674612402915955, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.841796875, + "rewards/tag_count_reward/std": 0.25722646713256836, + "step": 625 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.169921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1981.0, + "completions/mean_length": 965.78515625, + "completions/mean_terminated_length": 744.2493896484375, + "completions/min_length": 128.0, + "completions/min_terminated_length": 128.0, + "epoch": 0.2137065801826406, + "grad_norm": 2.918363094329834, + "kl": 1.091796875, + "learning_rate": 9.650500170704127e-07, + "loss": 0.3223, + "num_tokens": 413062593.0, + "reward": 1.00048828125, + "reward_std": 0.3051578402519226, + "rewards/accuracy_reward/mean": 0.12890625, + "rewards/accuracy_reward/std": 0.33542385697364807, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.87158203125, + "rewards/tag_count_reward/std": 0.23195762932300568, + "step": 626 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.21484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1790.0, + "completions/mean_length": 1026.69140625, + "completions/mean_terminated_length": 747.2288208007812, + "completions/min_length": 121.0, + "completions/min_terminated_length": 121.0, + "epoch": 0.21404796449603142, + "grad_norm": 1.993016004562378, + "kl": 1.376953125, + "learning_rate": 9.648425731008884e-07, + "loss": 0.3599, + "num_tokens": 413666819.0, + "reward": 0.90771484375, + "reward_std": 0.2868208587169647, + "rewards/accuracy_reward/mean": 0.06451612710952759, + "rewards/accuracy_reward/std": 0.2459181249141693, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.84521484375, + "rewards/tag_count_reward/std": 0.263744592666626, + "step": 627 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.212890625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1949.0, + "completions/mean_length": 996.869140625, + "completions/mean_terminated_length": 712.5682373046875, + "completions/min_length": 182.0, + "completions/min_terminated_length": 182.0, + "epoch": 0.2143893488094222, + "grad_norm": 9.06557846069336, + "kl": 2.041015625, + "learning_rate": 9.646345403382073e-07, + "loss": 0.4058, + "num_tokens": 414256832.0, + "reward": 0.8994140625, + "reward_std": 0.2883273959159851, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8505859375, + "rewards/tag_count_reward/std": 0.25678586959838867, + "step": 628 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.208984375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1975.0, + "completions/mean_length": 988.466796875, + "completions/mean_terminated_length": 708.540771484375, + "completions/min_length": 161.0, + "completions/min_terminated_length": 161.0, + "epoch": 0.21473073312281302, + "grad_norm": 9.490395545959473, + "kl": 1.888671875, + "learning_rate": 9.644259190776339e-07, + "loss": 0.4092, + "num_tokens": 414846015.0, + "reward": 0.8935546875, + "reward_std": 0.317560076713562, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8369140625, + "rewards/tag_count_reward/std": 0.26033368706703186, + "step": 629 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.27734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1917.0, + "completions/mean_length": 1080.869140625, + "completions/mean_terminated_length": 709.7000122070312, + "completions/min_length": 108.0, + "completions/min_terminated_length": 108.0, + "epoch": 0.2150721174362038, + "grad_norm": 5.034303665161133, + "kl": 1.919921875, + "learning_rate": 9.642167096152678e-07, + "loss": 0.4097, + "num_tokens": 415472380.0, + "reward": 0.86279296875, + "reward_std": 0.3311331570148468, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.81005859375, + "rewards/tag_count_reward/std": 0.27167558670043945, + "step": 630 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.275390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1978.0, + "completions/mean_length": 1102.31640625, + "completions/mean_terminated_length": 742.9056396484375, + "completions/min_length": 228.0, + "completions/min_terminated_length": 228.0, + "epoch": 0.21541350174959462, + "grad_norm": 3.3463125228881836, + "kl": 1.89453125, + "learning_rate": 9.640069122480437e-07, + "loss": 0.419, + "num_tokens": 416111182.0, + "reward": 0.849609375, + "reward_std": 0.3056153655052185, + "rewards/accuracy_reward/mean": 0.03515625, + "rewards/accuracy_reward/std": 0.1843547374010086, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.814453125, + "rewards/tag_count_reward/std": 0.27581244707107544, + "step": 631 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.255859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1996.0, + "completions/mean_length": 1081.791015625, + "completions/mean_terminated_length": 749.5774536132812, + "completions/min_length": 217.0, + "completions/min_terminated_length": 217.0, + "epoch": 0.2157548860629854, + "grad_norm": 2.0118744373321533, + "kl": 1.451171875, + "learning_rate": 9.637965272737305e-07, + "loss": 0.3259, + "num_tokens": 416742259.0, + "reward": 0.95458984375, + "reward_std": 0.3619065284729004, + "rewards/accuracy_reward/mean": 0.125, + "rewards/accuracy_reward/std": 0.3310423493385315, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.82958984375, + "rewards/tag_count_reward/std": 0.27249738574028015, + "step": 632 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.275390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2010.0, + "completions/mean_length": 1102.6484375, + "completions/mean_terminated_length": 743.3638916015625, + "completions/min_length": 224.0, + "completions/min_terminated_length": 224.0, + "epoch": 0.21609627037637621, + "grad_norm": 3.584123373031616, + "kl": 1.57421875, + "learning_rate": 9.635855549909314e-07, + "loss": 0.4054, + "num_tokens": 417380319.0, + "reward": 0.84814453125, + "reward_std": 0.30755841732025146, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.80712890625, + "rewards/tag_count_reward/std": 0.2745445668697357, + "step": 633 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.208984375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 981.1484375, + "completions/mean_terminated_length": 699.2889404296875, + "completions/min_length": 168.0, + "completions/min_terminated_length": 168.0, + "epoch": 0.216437654689767, + "grad_norm": 3.4458224773406982, + "kl": 1.330078125, + "learning_rate": 9.63373995699083e-07, + "loss": 0.3272, + "num_tokens": 417955531.0, + "reward": 0.92431640625, + "reward_std": 0.29383552074432373, + "rewards/accuracy_reward/mean": 0.08266129344701767, + "rewards/accuracy_reward/std": 0.2756476104259491, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.84423828125, + "rewards/tag_count_reward/std": 0.26036444306373596, + "step": 634 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.25, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2041.0, + "completions/mean_length": 1077.49609375, + "completions/mean_terminated_length": 753.9948120117188, + "completions/min_length": 209.0, + "completions/min_terminated_length": 209.0, + "epoch": 0.21677903900315781, + "grad_norm": 1.7347800731658936, + "kl": 1.4765625, + "learning_rate": 9.631618496984546e-07, + "loss": 0.3759, + "num_tokens": 418586489.0, + "reward": 0.84375, + "reward_std": 0.2716478109359741, + "rewards/accuracy_reward/mean": 0.021484375, + "rewards/accuracy_reward/std": 0.14513419568538666, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.822265625, + "rewards/tag_count_reward/std": 0.2702690064907074, + "step": 635 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.27734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1909.0, + "completions/mean_length": 1107.6015625, + "completions/mean_terminated_length": 746.69189453125, + "completions/min_length": 126.0, + "completions/min_terminated_length": 126.0, + "epoch": 0.2171204233165486, + "grad_norm": 2.8225364685058594, + "kl": 1.798828125, + "learning_rate": 9.62949117290149e-07, + "loss": 0.4617, + "num_tokens": 419229645.0, + "reward": 0.8427734375, + "reward_std": 0.3390008211135864, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.7919921875, + "rewards/tag_count_reward/std": 0.289990097284317, + "step": 636 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.34375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1986.0, + "completions/mean_length": 1199.63671875, + "completions/mean_terminated_length": 755.2559814453125, + "completions/min_length": 218.0, + "completions/min_terminated_length": 218.0, + "epoch": 0.2174618076299394, + "grad_norm": 3.7464311122894287, + "kl": 1.8359375, + "learning_rate": 9.627357987761007e-07, + "loss": 0.4191, + "num_tokens": 419923651.0, + "reward": 0.826171875, + "reward_std": 0.32486432790756226, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.78125, + "rewards/tag_count_reward/std": 0.2905053198337555, + "step": 637 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.37109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1828.0, + "completions/mean_length": 1204.205078125, + "completions/mean_terminated_length": 706.3136596679688, + "completions/min_length": 195.0, + "completions/min_terminated_length": 195.0, + "epoch": 0.2178031919433302, + "grad_norm": 4.913912296295166, + "kl": 2.041015625, + "learning_rate": 9.625218944590763e-07, + "loss": 0.4591, + "num_tokens": 420608700.0, + "reward": 0.8134765625, + "reward_std": 0.3354353904724121, + "rewards/accuracy_reward/mean": 0.05443548411130905, + "rewards/accuracy_reward/std": 0.227104052901268, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.7607421875, + "rewards/tag_count_reward/std": 0.30549731850624084, + "step": 638 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.32421875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2012.0, + "completions/mean_length": 1095.244140625, + "completions/mean_terminated_length": 638.1416015625, + "completions/min_length": 168.0, + "completions/min_terminated_length": 168.0, + "epoch": 0.218144576256721, + "grad_norm": 3.983170509338379, + "kl": 2.02734375, + "learning_rate": 9.623074046426744e-07, + "loss": 0.4582, + "num_tokens": 421242665.0, + "reward": 0.8583984375, + "reward_std": 0.3076876401901245, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.7880859375, + "rewards/tag_count_reward/std": 0.2905299961566925, + "step": 639 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.337890625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1799.0, + "completions/mean_length": 1141.359375, + "completions/mean_terminated_length": 678.678466796875, + "completions/min_length": 218.0, + "completions/min_terminated_length": 218.0, + "epoch": 0.2184859605701118, + "grad_norm": 2.0060925483703613, + "kl": 2.048828125, + "learning_rate": 9.620923296313234e-07, + "loss": 0.4543, + "num_tokens": 421902817.0, + "reward": 0.84765625, + "reward_std": 0.35959500074386597, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.7578125, + "rewards/tag_count_reward/std": 0.304784893989563, + "step": 640 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1801.0, + "completions/mean_length": 1226.77734375, + "completions/mean_terminated_length": 734.0437622070312, + "completions/min_length": 219.0, + "completions/min_terminated_length": 219.0, + "epoch": 0.2188273448835026, + "grad_norm": 2.325049877166748, + "kl": 2.037109375, + "learning_rate": 9.618766697302835e-07, + "loss": 0.4852, + "num_tokens": 422605567.0, + "reward": 0.78125, + "reward_std": 0.3310891389846802, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.734375, + "rewards/tag_count_reward/std": 0.3104507029056549, + "step": 641 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.26171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1981.0, + "completions/mean_length": 1047.201171875, + "completions/mean_terminated_length": 692.4205932617188, + "completions/min_length": 206.0, + "completions/min_terminated_length": 206.0, + "epoch": 0.2191687291968934, + "grad_norm": 4.015636920928955, + "kl": 1.654296875, + "learning_rate": 9.616604252456437e-07, + "loss": 0.4472, + "num_tokens": 423219302.0, + "reward": 0.84375, + "reward_std": 0.32642629742622375, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.791015625, + "rewards/tag_count_reward/std": 0.29473087191581726, + "step": 642 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.271484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2015.0, + "completions/mean_length": 1144.35546875, + "completions/mean_terminated_length": 807.6085815429688, + "completions/min_length": 216.0, + "completions/min_terminated_length": 216.0, + "epoch": 0.2195101135102842, + "grad_norm": 3.6396005153656006, + "kl": 1.47265625, + "learning_rate": 9.614435964843245e-07, + "loss": 0.3916, + "num_tokens": 423884940.0, + "reward": 0.81640625, + "reward_std": 0.3465976119041443, + "rewards/accuracy_reward/mean": 0.05443548411130905, + "rewards/accuracy_reward/std": 0.227104052901268, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.763671875, + "rewards/tag_count_reward/std": 0.3081708550453186, + "step": 643 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2890625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1986.0, + "completions/mean_length": 1122.044921875, + "completions/mean_terminated_length": 745.5577392578125, + "completions/min_length": 169.0, + "completions/min_terminated_length": 169.0, + "epoch": 0.219851497823675, + "grad_norm": 4.020759105682373, + "kl": 1.462890625, + "learning_rate": 9.612261837540738e-07, + "loss": 0.4117, + "num_tokens": 424533427.0, + "reward": 0.83837890625, + "reward_std": 0.32647258043289185, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.78369140625, + "rewards/tag_count_reward/std": 0.29997169971466064, + "step": 644 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.412109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1463.0, + "completions/mean_length": 1249.642578125, + "completions/mean_terminated_length": 689.9966430664062, + "completions/min_length": 144.0, + "completions/min_terminated_length": 144.0, + "epoch": 0.2201928821370658, + "grad_norm": 3.8461523056030273, + "kl": 1.94140625, + "learning_rate": 9.610081873634696e-07, + "loss": 0.4567, + "num_tokens": 425248124.0, + "reward": 0.73583984375, + "reward_std": 0.3610597252845764, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.69677734375, + "rewards/tag_count_reward/std": 0.3291517496109009, + "step": 645 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.37109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 1224.62890625, + "completions/mean_terminated_length": 738.788818359375, + "completions/min_length": 173.0, + "completions/min_terminated_length": 173.0, + "epoch": 0.2205342664504566, + "grad_norm": 3.524672746658325, + "kl": 1.8984375, + "learning_rate": 9.607896076219181e-07, + "loss": 0.4526, + "num_tokens": 425963998.0, + "reward": 0.75634765625, + "reward_std": 0.336562842130661, + "rewards/accuracy_reward/mean": 0.037109375, + "rewards/accuracy_reward/std": 0.18921469151973724, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.71923828125, + "rewards/tag_count_reward/std": 0.3142194449901581, + "step": 646 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.392578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1799.0, + "completions/mean_length": 1248.44921875, + "completions/mean_terminated_length": 731.69775390625, + "completions/min_length": 224.0, + "completions/min_terminated_length": 224.0, + "epoch": 0.2208756507638474, + "grad_norm": 2.5784406661987305, + "kl": 1.724609375, + "learning_rate": 9.605704448396529e-07, + "loss": 0.4351, + "num_tokens": 426684164.0, + "reward": 0.76806640625, + "reward_std": 0.3576868176460266, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.69775390625, + "rewards/tag_count_reward/std": 0.31873929500579834, + "step": 647 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.326171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1911.0, + "completions/mean_length": 1166.34765625, + "completions/mean_terminated_length": 739.5768432617188, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "epoch": 0.2212170350772382, + "grad_norm": 4.20598840713501, + "kl": 1.849609375, + "learning_rate": 9.603506993277354e-07, + "loss": 0.4638, + "num_tokens": 427353270.0, + "reward": 0.79931640625, + "reward_std": 0.3777962327003479, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.73291015625, + "rewards/tag_count_reward/std": 0.31796491146087646, + "step": 648 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1823.0, + "completions/mean_length": 1143.755859375, + "completions/mean_terminated_length": 732.73583984375, + "completions/min_length": 218.0, + "completions/min_terminated_length": 218.0, + "epoch": 0.221558419390629, + "grad_norm": 3.2356879711151123, + "kl": 1.5703125, + "learning_rate": 9.601303713980545e-07, + "loss": 0.4345, + "num_tokens": 428012521.0, + "reward": 0.79541015625, + "reward_std": 0.3835356831550598, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.74462890625, + "rewards/tag_count_reward/std": 0.31334546208381653, + "step": 649 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.302734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2030.0, + "completions/mean_length": 1155.529296875, + "completions/mean_terminated_length": 768.0420532226562, + "completions/min_length": 192.0, + "completions/min_terminated_length": 192.0, + "epoch": 0.2218998037040198, + "grad_norm": 4.234147071838379, + "kl": 1.3828125, + "learning_rate": 9.599094613633255e-07, + "loss": 0.3902, + "num_tokens": 428684888.0, + "reward": 0.8271484375, + "reward_std": 0.38090699911117554, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.7431640625, + "rewards/tag_count_reward/std": 0.30958616733551025, + "step": 650 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.330078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1998.0, + "completions/mean_length": 1201.9921875, + "completions/mean_terminated_length": 785.154541015625, + "completions/min_length": 267.0, + "completions/min_terminated_length": 267.0, + "epoch": 0.2222411880174106, + "grad_norm": 3.7783749103546143, + "kl": 1.68359375, + "learning_rate": 9.596879695370894e-07, + "loss": 0.4232, + "num_tokens": 429373508.0, + "reward": 0.78271484375, + "reward_std": 0.38518527150154114, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.72021484375, + "rewards/tag_count_reward/std": 0.32427331805229187, + "step": 651 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.2890625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1840.0, + "completions/mean_length": 1102.439453125, + "completions/mean_terminated_length": 717.9807739257812, + "completions/min_length": 155.0, + "completions/min_terminated_length": 155.0, + "epoch": 0.2225825723308014, + "grad_norm": 5.931146144866943, + "kl": 1.54296875, + "learning_rate": 9.594658962337134e-07, + "loss": 0.4444, + "num_tokens": 430018293.0, + "reward": 0.81689453125, + "reward_std": 0.3459562659263611, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.76416015625, + "rewards/tag_count_reward/std": 0.3099297285079956, + "step": 652 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.31640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2023.0, + "completions/mean_length": 1171.966796875, + "completions/mean_terminated_length": 766.4885864257812, + "completions/min_length": 151.0, + "completions/min_terminated_length": 151.0, + "epoch": 0.2229239566441922, + "grad_norm": 3.5709598064422607, + "kl": 1.8046875, + "learning_rate": 9.592432417683903e-07, + "loss": 0.4129, + "num_tokens": 430697892.0, + "reward": 0.7685546875, + "reward_std": 0.36317336559295654, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.7041015625, + "rewards/tag_count_reward/std": 0.3233308494091034, + "step": 653 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.322265625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1935.0, + "completions/mean_length": 1180.69140625, + "completions/mean_terminated_length": 768.2824096679688, + "completions/min_length": 215.0, + "completions/min_terminated_length": 215.0, + "epoch": 0.223265340957583, + "grad_norm": 5.544654846191406, + "kl": 1.74609375, + "learning_rate": 9.59020006457137e-07, + "loss": 0.4349, + "num_tokens": 431384006.0, + "reward": 0.771484375, + "reward_std": 0.36828142404556274, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.720703125, + "rewards/tag_count_reward/std": 0.3233736753463745, + "step": 654 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.275390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1902.0, + "completions/mean_length": 1096.169921875, + "completions/mean_terminated_length": 734.4231567382812, + "completions/min_length": 174.0, + "completions/min_terminated_length": 174.0, + "epoch": 0.2236067252709738, + "grad_norm": 6.116422176361084, + "kl": 2.353515625, + "learning_rate": 9.587961906167952e-07, + "loss": 0.4265, + "num_tokens": 432017821.0, + "reward": 0.8349609375, + "reward_std": 0.3506982922554016, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.7607421875, + "rewards/tag_count_reward/std": 0.3090795576572418, + "step": 655 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.24609375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1882.0, + "completions/mean_length": 1036.408203125, + "completions/mean_terminated_length": 706.199462890625, + "completions/min_length": 159.0, + "completions/min_terminated_length": 159.0, + "epoch": 0.2239481095843646, + "grad_norm": 6.848345756530762, + "kl": 2.25390625, + "learning_rate": 9.585717945650307e-07, + "loss": 0.4598, + "num_tokens": 432624126.0, + "reward": 0.89990234375, + "reward_std": 0.4077647030353546, + "rewards/accuracy_reward/mean": 0.140625, + "rewards/accuracy_reward/std": 0.3479743003845215, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.75927734375, + "rewards/tag_count_reward/std": 0.31208041310310364, + "step": 656 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.31640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1953.0, + "completions/mean_length": 1143.375, + "completions/mean_terminated_length": 724.662841796875, + "completions/min_length": 196.0, + "completions/min_terminated_length": 196.0, + "epoch": 0.2242894938977554, + "grad_norm": 9.063117980957031, + "kl": 2.5, + "learning_rate": 9.583468186203326e-07, + "loss": 0.5099, + "num_tokens": 433281934.0, + "reward": 0.77880859375, + "reward_std": 0.3714064955711365, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.72216796875, + "rewards/tag_count_reward/std": 0.32217732071876526, + "step": 657 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.322265625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1942.0, + "completions/mean_length": 1146.294921875, + "completions/mean_terminated_length": 717.5302124023438, + "completions/min_length": 160.0, + "completions/min_terminated_length": 160.0, + "epoch": 0.2246308782111462, + "grad_norm": 16.551767349243164, + "kl": 3.5390625, + "learning_rate": 9.581212631020132e-07, + "loss": 0.5075, + "num_tokens": 433937845.0, + "reward": 0.765625, + "reward_std": 0.35465750098228455, + "rewards/accuracy_reward/mean": 0.038306452333927155, + "rewards/accuracy_reward/std": 0.19212883710861206, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.728515625, + "rewards/tag_count_reward/std": 0.3186587393283844, + "step": 658 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.3046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1755.0, + "completions/mean_length": 1129.478515625, + "completions/mean_terminated_length": 726.9803466796875, + "completions/min_length": 193.0, + "completions/min_terminated_length": 193.0, + "epoch": 0.224972262524537, + "grad_norm": 10.934162139892578, + "kl": 3.0546875, + "learning_rate": 9.578951283302072e-07, + "loss": 0.5226, + "num_tokens": 434597802.0, + "reward": 0.779296875, + "reward_std": 0.3694903254508972, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.73046875, + "rewards/tag_count_reward/std": 0.3218393921852112, + "step": 659 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.263671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 1107.08203125, + "completions/mean_terminated_length": 770.1484985351562, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.2253136468379278, + "grad_norm": 10.84688949584961, + "kl": 3.5546875, + "learning_rate": 9.576684146258715e-07, + "loss": 0.4756, + "num_tokens": 435238276.0, + "reward": 0.771484375, + "reward_std": 0.3382296562194824, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.732421875, + "rewards/tag_count_reward/std": 0.3154278099536896, + "step": 660 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.205078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2041.0, + "completions/mean_length": 1014.0, + "completions/mean_terminated_length": 747.2432250976562, + "completions/min_length": 229.0, + "completions/min_terminated_length": 229.0, + "epoch": 0.2256550311513186, + "grad_norm": 9.455982208251953, + "kl": 3.6484375, + "learning_rate": 9.574411223107849e-07, + "loss": 0.4937, + "num_tokens": 435835508.0, + "reward": 0.82861328125, + "reward_std": 0.37730008363723755, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.75830078125, + "rewards/tag_count_reward/std": 0.30856090784072876, + "step": 661 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.17578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1966.0, + "completions/mean_length": 984.41796875, + "completions/mean_terminated_length": 757.5877075195312, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "epoch": 0.2259964154647094, + "grad_norm": 55.217586517333984, + "kl": 5.5546875, + "learning_rate": 9.572132517075472e-07, + "loss": 0.5353, + "num_tokens": 436417178.0, + "reward": 0.8818359375, + "reward_std": 0.38336285948753357, + "rewards/accuracy_reward/mean": 0.103515625, + "rewards/accuracy_reward/std": 0.30492907762527466, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.7783203125, + "rewards/tag_count_reward/std": 0.29787030816078186, + "step": 662 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.154296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1991.0, + "completions/mean_length": 998.3984375, + "completions/mean_terminated_length": 806.9006958007812, + "completions/min_length": 177.0, + "completions/min_terminated_length": 177.0, + "epoch": 0.2263377997781002, + "grad_norm": 8.337776184082031, + "kl": 4.05859375, + "learning_rate": 9.56984803139579e-07, + "loss": 0.4669, + "num_tokens": 437012886.0, + "reward": 0.8271484375, + "reward_std": 0.3475971519947052, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.7841796875, + "rewards/tag_count_reward/std": 0.29185569286346436, + "step": 663 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.138671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2012.0, + "completions/mean_length": 954.8515625, + "completions/mean_terminated_length": 778.857177734375, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "epoch": 0.226679184091491, + "grad_norm": 17.617353439331055, + "kl": 4.45703125, + "learning_rate": 9.567557769311213e-07, + "loss": 0.4034, + "num_tokens": 437574442.0, + "reward": 0.8857421875, + "reward_std": 0.33483001589775085, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8037109375, + "rewards/tag_count_reward/std": 0.27854791283607483, + "step": 664 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.076171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 884.009765625, + "completions/mean_terminated_length": 788.0359497070312, + "completions/min_length": 177.0, + "completions/min_terminated_length": 177.0, + "epoch": 0.2270205684048818, + "grad_norm": 14.468287467956543, + "kl": 4.265625, + "learning_rate": 9.565261734072346e-07, + "loss": 0.4193, + "num_tokens": 438108351.0, + "reward": 0.8193359375, + "reward_std": 0.32358604669570923, + "rewards/accuracy_reward/mean": 0.025390625, + "rewards/accuracy_reward/std": 0.15746226906776428, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.7939453125, + "rewards/tag_count_reward/std": 0.2884306311607361, + "step": 665 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1992.0, + "completions/mean_length": 837.427734375, + "completions/mean_terminated_length": 734.8368530273438, + "completions/min_length": 140.0, + "completions/min_terminated_length": 140.0, + "epoch": 0.2273619527182726, + "grad_norm": 12.217328071594238, + "kl": 3.38671875, + "learning_rate": 9.562959928937999e-07, + "loss": 0.3894, + "num_tokens": 438616250.0, + "reward": 0.8662109375, + "reward_std": 0.3066549599170685, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8232421875, + "rewards/tag_count_reward/std": 0.27226102352142334, + "step": 666 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1991.0, + "completions/mean_length": 795.8984375, + "completions/mean_terminated_length": 723.4627685546875, + "completions/min_length": 110.0, + "completions/min_terminated_length": 110.0, + "epoch": 0.2277033370316634, + "grad_norm": 8.62721061706543, + "kl": 3.6328125, + "learning_rate": 9.560652357175157e-07, + "loss": 0.3485, + "num_tokens": 439102582.0, + "reward": 0.93359375, + "reward_std": 0.3151872754096985, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.841796875, + "rewards/tag_count_reward/std": 0.2600637674331665, + "step": 667 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1999.0, + "completions/mean_length": 775.390625, + "completions/mean_terminated_length": 723.6585083007812, + "completions/min_length": 70.0, + "completions/min_terminated_length": 70.0, + "epoch": 0.22804472134505419, + "grad_norm": 12.339641571044922, + "kl": 3.54296875, + "learning_rate": 9.558339022058995e-07, + "loss": 0.3654, + "num_tokens": 439572894.0, + "reward": 0.921875, + "reward_std": 0.3119737505912781, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.83984375, + "rewards/tag_count_reward/std": 0.25744181871414185, + "step": 668 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1943.0, + "completions/mean_length": 812.435546875, + "completions/mean_terminated_length": 762.2093505859375, + "completions/min_length": 75.0, + "completions/min_terminated_length": 75.0, + "epoch": 0.228386105658445, + "grad_norm": 15.873409271240234, + "kl": 3.3125, + "learning_rate": 9.556019926872874e-07, + "loss": 0.3952, + "num_tokens": 440066541.0, + "reward": 0.8818359375, + "reward_std": 0.3047889173030853, + "rewards/accuracy_reward/mean": 0.060483869165182114, + "rewards/accuracy_reward/std": 0.2386218160390854, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8232421875, + "rewards/tag_count_reward/std": 0.27181142568588257, + "step": 669 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 736.40625, + "completions/mean_terminated_length": 707.6087646484375, + "completions/min_length": 128.0, + "completions/min_terminated_length": 128.0, + "epoch": 0.22872748997183578, + "grad_norm": 5.676901340484619, + "kl": 3.48046875, + "learning_rate": 9.553695074908321e-07, + "loss": 0.2961, + "num_tokens": 440525437.0, + "reward": 0.94775390625, + "reward_std": 0.3289104402065277, + "rewards/accuracy_reward/mean": 0.099609375, + "rewards/accuracy_reward/std": 0.29977133870124817, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.84619140625, + "rewards/tag_count_reward/std": 0.25632917881011963, + "step": 670 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 722.0, + "completions/mean_terminated_length": 681.9798583984375, + "completions/min_length": 107.0, + "completions/min_terminated_length": 107.0, + "epoch": 0.2290688742852266, + "grad_norm": 5.196404933929443, + "kl": 4.921875, + "learning_rate": 9.55136446946504e-07, + "loss": 0.4483, + "num_tokens": 440963101.0, + "reward": 0.91064453125, + "reward_std": 0.3597055971622467, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.81298828125, + "rewards/tag_count_reward/std": 0.27857664227485657, + "step": 671 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2004.0, + "completions/mean_length": 827.990234375, + "completions/mean_terminated_length": 783.5364379882812, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.22941025859861738, + "grad_norm": 23.459131240844727, + "kl": 7.03125, + "learning_rate": 9.549028113850903e-07, + "loss": 0.4704, + "num_tokens": 441464904.0, + "reward": 0.775390625, + "reward_std": 0.33348000049591064, + "rewards/accuracy_reward/mean": 0.037109375, + "rewards/accuracy_reward/std": 0.18921469151973724, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.73828125, + "rewards/tag_count_reward/std": 0.3066602051258087, + "step": 672 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2028.0, + "completions/mean_length": 785.119140625, + "completions/mean_terminated_length": 747.0040283203125, + "completions/min_length": 118.0, + "completions/min_terminated_length": 118.0, + "epoch": 0.2297516429120082, + "grad_norm": 21.86775779724121, + "kl": 7.875, + "learning_rate": 9.54668601138193e-07, + "loss": 0.5296, + "num_tokens": 441944709.0, + "reward": 0.828125, + "reward_std": 0.33433833718299866, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.7734375, + "rewards/tag_count_reward/std": 0.28228431940078735, + "step": 673 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 785.490234375, + "completions/mean_terminated_length": 744.7640991210938, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "epoch": 0.23009302722539898, + "grad_norm": 13.236749649047852, + "kl": 7.3671875, + "learning_rate": 9.544338165382318e-07, + "loss": 0.5494, + "num_tokens": 442423712.0, + "reward": 0.7939453125, + "reward_std": 0.33076488971710205, + "rewards/accuracy_reward/mean": 0.06653226166963577, + "rewards/accuracy_reward/std": 0.24946178495883942, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.7294921875, + "rewards/tag_count_reward/std": 0.29851120710372925, + "step": 674 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2011.0, + "completions/mean_length": 800.494140625, + "completions/mean_terminated_length": 765.4236450195312, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "epoch": 0.2304344115387898, + "grad_norm": 10.939888954162598, + "kl": 6.359375, + "learning_rate": 9.541984579184399e-07, + "loss": 0.4581, + "num_tokens": 442914301.0, + "reward": 0.7734375, + "reward_std": 0.35030168294906616, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.724609375, + "rewards/tag_count_reward/std": 0.3034211993217468, + "step": 675 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1940.0, + "completions/mean_length": 797.484375, + "completions/mean_terminated_length": 759.742431640625, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.23077579585218058, + "grad_norm": 4.543842792510986, + "kl": 4.94140625, + "learning_rate": 9.539625256128658e-07, + "loss": 0.3953, + "num_tokens": 443401701.0, + "reward": 0.80859375, + "reward_std": 0.3620191514492035, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.7265625, + "rewards/tag_count_reward/std": 0.29747387766838074, + "step": 676 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1967.0, + "completions/mean_length": 751.935546875, + "completions/mean_terminated_length": 726.1175537109375, + "completions/min_length": 4.0, + "completions/min_terminated_length": 4.0, + "epoch": 0.2311171801655714, + "grad_norm": 5.480804920196533, + "kl": 4.234375, + "learning_rate": 9.537260199563723e-07, + "loss": 0.3736, + "num_tokens": 443870196.0, + "reward": 0.8095703125, + "reward_std": 0.3241175413131714, + "rewards/accuracy_reward/mean": 0.033203125, + "rewards/accuracy_reward/std": 0.17934183776378632, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.7763671875, + "rewards/tag_count_reward/std": 0.28289130330085754, + "step": 677 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2005.0, + "completions/mean_length": 738.060546875, + "completions/mean_terminated_length": 717.2678833007812, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "epoch": 0.23145856447896218, + "grad_norm": 8.061043739318848, + "kl": 3.640625, + "learning_rate": 9.534889412846361e-07, + "loss": 0.324, + "num_tokens": 444319731.0, + "reward": 0.86083984375, + "reward_std": 0.33779412508010864, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.79833984375, + "rewards/tag_count_reward/std": 0.2788749039173126, + "step": 678 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1977.0, + "completions/mean_length": 767.890625, + "completions/mean_terminated_length": 737.1680297851562, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "epoch": 0.231799948792353, + "grad_norm": 10.127677917480469, + "kl": 3.7421875, + "learning_rate": 9.532512899341467e-07, + "loss": 0.3546, + "num_tokens": 444791643.0, + "reward": 0.859375, + "reward_std": 0.3290579319000244, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.791015625, + "rewards/tag_count_reward/std": 0.28070247173309326, + "step": 679 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.005859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2028.0, + "completions/mean_length": 734.447265625, + "completions/mean_terminated_length": 726.705322265625, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "epoch": 0.23214133310574378, + "grad_norm": 10.456915855407715, + "kl": 3.46875, + "learning_rate": 9.53013066242207e-07, + "loss": 0.3382, + "num_tokens": 445253072.0, + "reward": 0.88427734375, + "reward_std": 0.32650578022003174, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.81005859375, + "rewards/tag_count_reward/std": 0.2712250053882599, + "step": 680 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.009765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1891.0, + "completions/mean_length": 718.912109375, + "completions/mean_terminated_length": 705.8047485351562, + "completions/min_length": 122.0, + "completions/min_terminated_length": 122.0, + "epoch": 0.2324827174191346, + "grad_norm": 3.8908488750457764, + "kl": 4.5078125, + "learning_rate": 9.527742705469318e-07, + "loss": 0.3677, + "num_tokens": 445695043.0, + "reward": 0.900390625, + "reward_std": 0.36471623182296753, + "rewards/accuracy_reward/mean": 0.099609375, + "rewards/accuracy_reward/std": 0.29977133870124817, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.798828125, + "rewards/tag_count_reward/std": 0.2785702049732208, + "step": 681 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.009765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2002.0, + "completions/mean_length": 735.57421875, + "completions/mean_terminated_length": 722.6311645507812, + "completions/min_length": 89.0, + "completions/min_terminated_length": 89.0, + "epoch": 0.23282410173252538, + "grad_norm": 4.040297985076904, + "kl": 5.4765625, + "learning_rate": 9.525349031872481e-07, + "loss": 0.3961, + "num_tokens": 446147449.0, + "reward": 0.80419921875, + "reward_std": 0.3184512257575989, + "rewards/accuracy_reward/mean": 0.03515625, + "rewards/accuracy_reward/std": 0.1843547374010086, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.76904296875, + "rewards/tag_count_reward/std": 0.28022506833076477, + "step": 682 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2024.0, + "completions/mean_length": 717.15625, + "completions/mean_terminated_length": 693.3439331054688, + "completions/min_length": 32.0, + "completions/min_terminated_length": 32.0, + "epoch": 0.2331654860459162, + "grad_norm": 4.727411270141602, + "kl": 5.390625, + "learning_rate": 9.52294964502894e-07, + "loss": 0.3808, + "num_tokens": 446591257.0, + "reward": 0.7978515625, + "reward_std": 0.33729878067970276, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.7373046875, + "rewards/tag_count_reward/std": 0.2872621417045593, + "step": 683 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1986.0, + "completions/mean_length": 715.095703125, + "completions/mean_terminated_length": 709.86865234375, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "epoch": 0.23350687035930698, + "grad_norm": 3.564438581466675, + "kl": 5.296875, + "learning_rate": 9.520544548344184e-07, + "loss": 0.3726, + "num_tokens": 447031610.0, + "reward": 0.84130859375, + "reward_std": 0.3505712151527405, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.76513671875, + "rewards/tag_count_reward/std": 0.2847912311553955, + "step": 684 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1864.0, + "completions/mean_length": 754.017578125, + "completions/mean_terminated_length": 743.8287353515625, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "epoch": 0.2338482546726978, + "grad_norm": 9.209820747375488, + "kl": 5.546875, + "learning_rate": 9.51813374523181e-07, + "loss": 0.3173, + "num_tokens": 447496787.0, + "reward": 0.80517578125, + "reward_std": 0.3297889232635498, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.76220703125, + "rewards/tag_count_reward/std": 0.2806067168712616, + "step": 685 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1987.0, + "completions/mean_length": 728.1796875, + "completions/mean_terminated_length": 712.5296630859375, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "epoch": 0.23418963898608858, + "grad_norm": 9.223282814025879, + "kl": 6.0546875, + "learning_rate": 9.515717239113511e-07, + "loss": 0.3745, + "num_tokens": 447945871.0, + "reward": 0.78271484375, + "reward_std": 0.36198490858078003, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.72998046875, + "rewards/tag_count_reward/std": 0.303219199180603, + "step": 686 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1909.0, + "completions/mean_length": 728.1484375, + "completions/mean_terminated_length": 717.7559204101562, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.2345310232994794, + "grad_norm": 3.5982277393341064, + "kl": 4.69140625, + "learning_rate": 9.513295033419077e-07, + "loss": 0.33, + "num_tokens": 448398059.0, + "reward": 0.81884765625, + "reward_std": 0.3465069532394409, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.76806640625, + "rewards/tag_count_reward/std": 0.29015275835990906, + "step": 687 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 754.5390625, + "completions/mean_terminated_length": 744.3543090820312, + "completions/min_length": 96.0, + "completions/min_terminated_length": 96.0, + "epoch": 0.23487240761287018, + "grad_norm": 3.5190393924713135, + "kl": 4.75390625, + "learning_rate": 9.510867131586383e-07, + "loss": 0.3383, + "num_tokens": 448873087.0, + "reward": 0.77197265625, + "reward_std": 0.3334955871105194, + "rewards/accuracy_reward/mean": 0.03125, + "rewards/accuracy_reward/std": 0.17416280508041382, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.73876953125, + "rewards/tag_count_reward/std": 0.3012463450431824, + "step": 688 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.005859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1997.0, + "completions/mean_length": 736.884765625, + "completions/mean_terminated_length": 729.1572265625, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "epoch": 0.235213791926261, + "grad_norm": 4.137765407562256, + "kl": 4.7109375, + "learning_rate": 9.508433537061394e-07, + "loss": 0.3412, + "num_tokens": 449330356.0, + "reward": 0.78125, + "reward_std": 0.31054240465164185, + "rewards/accuracy_reward/mean": 0.02217741869390011, + "rewards/accuracy_reward/std": 0.14740893244743347, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.7578125, + "rewards/tag_count_reward/std": 0.29499658942222595, + "step": 689 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1898.0, + "completions/mean_length": 778.779296875, + "completions/mean_terminated_length": 740.4728393554688, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "epoch": 0.23555517623965178, + "grad_norm": 4.3230671882629395, + "kl": 5.7734375, + "learning_rate": 9.505994253298152e-07, + "loss": 0.4177, + "num_tokens": 449808403.0, + "reward": 0.79150390625, + "reward_std": 0.35842466354370117, + "rewards/accuracy_reward/mean": 0.060483869165182114, + "rewards/accuracy_reward/std": 0.2386218160390854, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.73095703125, + "rewards/tag_count_reward/std": 0.30003857612609863, + "step": 690 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.009765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1843.0, + "completions/mean_length": 707.78125, + "completions/mean_terminated_length": 694.5640869140625, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.2358965605530426, + "grad_norm": 2.4250447750091553, + "kl": 4.6171875, + "learning_rate": 9.503549283758773e-07, + "loss": 0.3016, + "num_tokens": 450247379.0, + "reward": 0.80224609375, + "reward_std": 0.3462855815887451, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.75146484375, + "rewards/tag_count_reward/std": 0.280868798494339, + "step": 691 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2018.0, + "completions/mean_length": 724.677734375, + "completions/mean_terminated_length": 708.9862060546875, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "epoch": 0.23623794486643337, + "grad_norm": 5.178401470184326, + "kl": 5.0703125, + "learning_rate": 9.501098631913446e-07, + "loss": 0.3659, + "num_tokens": 450699214.0, + "reward": 0.79150390625, + "reward_std": 0.3480370044708252, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.74853515625, + "rewards/tag_count_reward/std": 0.29488927125930786, + "step": 692 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1936.0, + "completions/mean_length": 800.224609375, + "completions/mean_terminated_length": 765.1465454101562, + "completions/min_length": 139.0, + "completions/min_terminated_length": 139.0, + "epoch": 0.2365793291798242, + "grad_norm": 2.476116895675659, + "kl": 5.4140625, + "learning_rate": 9.498642301240422e-07, + "loss": 0.3614, + "num_tokens": 451178609.0, + "reward": 0.78857421875, + "reward_std": 0.3727818727493286, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.72998046875, + "rewards/tag_count_reward/std": 0.29997488856315613, + "step": 693 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 809.4296875, + "completions/mean_terminated_length": 787.2683715820312, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.23692071349321497, + "grad_norm": 2.4830360412597656, + "kl": 5.6953125, + "learning_rate": 9.496180295226012e-07, + "loss": 0.3686, + "num_tokens": 451665549.0, + "reward": 0.7900390625, + "reward_std": 0.34522438049316406, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.7412109375, + "rewards/tag_count_reward/std": 0.2937224805355072, + "step": 694 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 754.28515625, + "completions/mean_terminated_length": 715.2394409179688, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.2372620978066058, + "grad_norm": 5.1372833251953125, + "kl": 5.6953125, + "learning_rate": 9.493712617364585e-07, + "loss": 0.3957, + "num_tokens": 452125343.0, + "reward": 0.81884765625, + "reward_std": 0.371033638715744, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.75244140625, + "rewards/tag_count_reward/std": 0.2985924184322357, + "step": 695 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1986.0, + "completions/mean_length": 769.015625, + "completions/mean_terminated_length": 711.591796875, + "completions/min_length": 7.0, + "completions/min_terminated_length": 7.0, + "epoch": 0.23760348211999657, + "grad_norm": 7.955462455749512, + "kl": 7.3828125, + "learning_rate": 9.491239271158558e-07, + "loss": 0.4834, + "num_tokens": 452598519.0, + "reward": 0.78466796875, + "reward_std": 0.38856804370880127, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.72216796875, + "rewards/tag_count_reward/std": 0.30541715025901794, + "step": 696 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 771.4140625, + "completions/mean_terminated_length": 714.097900390625, + "completions/min_length": 70.0, + "completions/min_terminated_length": 70.0, + "epoch": 0.2379448664333874, + "grad_norm": 5.789423942565918, + "kl": 6.9609375, + "learning_rate": 9.488760260118393e-07, + "loss": 0.4788, + "num_tokens": 453069643.0, + "reward": 0.82373046875, + "reward_std": 0.3601980209350586, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.73974609375, + "rewards/tag_count_reward/std": 0.29760515689849854, + "step": 697 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 814.333984375, + "completions/mean_terminated_length": 758.9448852539062, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.23828625074677817, + "grad_norm": 4.101593971252441, + "kl": 6.53125, + "learning_rate": 9.486275587762592e-07, + "loss": 0.463, + "num_tokens": 453560966.0, + "reward": 0.75732421875, + "reward_std": 0.32711267471313477, + "rewards/accuracy_reward/mean": 0.0234375, + "rewards/accuracy_reward/std": 0.15143637359142303, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.73193359375, + "rewards/tag_count_reward/std": 0.3033420741558075, + "step": 698 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.060546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 842.97265625, + "completions/mean_terminated_length": 765.3097534179688, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.23862763506016899, + "grad_norm": 2.990671157836914, + "kl": 6.6953125, + "learning_rate": 9.483785257617695e-07, + "loss": 0.4557, + "num_tokens": 454064728.0, + "reward": 0.82080078125, + "reward_std": 0.3680709898471832, + "rewards/accuracy_reward/mean": 0.06451612710952759, + "rewards/accuracy_reward/std": 0.2459181249141693, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.75439453125, + "rewards/tag_count_reward/std": 0.2872966229915619, + "step": 699 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2021.0, + "completions/mean_length": 770.794921875, + "completions/mean_terminated_length": 713.4509887695312, + "completions/min_length": 41.0, + "completions/min_terminated_length": 41.0, + "epoch": 0.23896901937355977, + "grad_norm": 5.022763729095459, + "kl": 5.5, + "learning_rate": 9.48128927321827e-07, + "loss": 0.4159, + "num_tokens": 454530015.0, + "reward": 0.8564453125, + "reward_std": 0.3755378723144531, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.7666015625, + "rewards/tag_count_reward/std": 0.28363341093063354, + "step": 700 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 848.177734375, + "completions/mean_terminated_length": 746.4978637695312, + "completions/min_length": 36.0, + "completions/min_terminated_length": 36.0, + "epoch": 0.23931040368695058, + "grad_norm": 9.848064422607422, + "kl": 5.9140625, + "learning_rate": 9.478787638106908e-07, + "loss": 0.4741, + "num_tokens": 455043306.0, + "reward": 0.869140625, + "reward_std": 0.3720466196537018, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.7890625, + "rewards/tag_count_reward/std": 0.2814164161682129, + "step": 701 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 802.310546875, + "completions/mean_terminated_length": 696.74365234375, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.2396517880003414, + "grad_norm": 10.3944673538208, + "kl": 6.125, + "learning_rate": 9.476280355834224e-07, + "loss": 0.5099, + "num_tokens": 455537001.0, + "reward": 0.818359375, + "reward_std": 0.3636171817779541, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.763671875, + "rewards/tag_count_reward/std": 0.29228267073631287, + "step": 702 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0703125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 817.974609375, + "completions/mean_terminated_length": 724.947509765625, + "completions/min_length": 69.0, + "completions/min_terminated_length": 69.0, + "epoch": 0.23999317231373218, + "grad_norm": 2.7264370918273926, + "kl": 6.1015625, + "learning_rate": 9.473767429958846e-07, + "loss": 0.4261, + "num_tokens": 456024540.0, + "reward": 0.78515625, + "reward_std": 0.335408091545105, + "rewards/accuracy_reward/mean": 0.029296875, + "rewards/accuracy_reward/std": 0.16880230605602264, + "rewards/format_reward/mean": 0.0078125, + "rewards/format_reward/std": 0.08812850713729858, + "rewards/tag_count_reward/mean": 0.748046875, + "rewards/tag_count_reward/std": 0.28151825070381165, + "step": 703 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 775.990234375, + "completions/mean_terminated_length": 705.1773681640625, + "completions/min_length": 21.0, + "completions/min_terminated_length": 21.0, + "epoch": 0.240334556627123, + "grad_norm": 5.046159267425537, + "kl": 7.0, + "learning_rate": 9.471248864047415e-07, + "loss": 0.4507, + "num_tokens": 456497527.0, + "reward": 0.81005859375, + "reward_std": 0.34700971841812134, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.76123046875, + "rewards/tag_count_reward/std": 0.2866840064525604, + "step": 704 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.056640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 800.5, + "completions/mean_terminated_length": 725.5983276367188, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "epoch": 0.24067594094051378, + "grad_norm": 9.976103782653809, + "kl": 8.1796875, + "learning_rate": 9.468724661674571e-07, + "loss": 0.5195, + "num_tokens": 456982023.0, + "reward": 0.88720703125, + "reward_std": 0.4148992896080017, + "rewards/accuracy_reward/mean": 0.12096773833036423, + "rewards/accuracy_reward/std": 0.32641899585723877, + "rewards/format_reward/mean": 0.0078125, + "rewards/format_reward/std": 0.08812850713729858, + "rewards/tag_count_reward/mean": 0.76220703125, + "rewards/tag_count_reward/std": 0.2891928553581238, + "step": 705 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.12109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2004.0, + "completions/mean_length": 884.802734375, + "completions/mean_terminated_length": 724.5400390625, + "completions/min_length": 35.0, + "completions/min_terminated_length": 35.0, + "epoch": 0.2410173252539046, + "grad_norm": 8.916975021362305, + "kl": 9.1171875, + "learning_rate": 9.466194826422961e-07, + "loss": 0.606, + "num_tokens": 457515922.0, + "reward": 0.8720703125, + "reward_std": 0.4169638752937317, + "rewards/accuracy_reward/mean": 0.12109375, + "rewards/accuracy_reward/std": 0.3265552520751953, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.7470703125, + "rewards/tag_count_reward/std": 0.3000182807445526, + "step": 706 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1973.0, + "completions/mean_length": 897.111328125, + "completions/mean_terminated_length": 732.6986694335938, + "completions/min_length": 75.0, + "completions/min_terminated_length": 75.0, + "epoch": 0.24135870956729538, + "grad_norm": 6.31559944152832, + "kl": 8.90625, + "learning_rate": 9.463659361883219e-07, + "loss": 0.6457, + "num_tokens": 458057579.0, + "reward": 0.796875, + "reward_std": 0.37265852093696594, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.7421875, + "rewards/tag_count_reward/std": 0.30155742168426514, + "step": 707 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.10546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 894.158203125, + "completions/mean_terminated_length": 758.11572265625, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "epoch": 0.2417000938806862, + "grad_norm": 2.506101131439209, + "kl": 7.1328125, + "learning_rate": 9.46111827165398e-07, + "loss": 0.5252, + "num_tokens": 458596572.0, + "reward": 0.77197265625, + "reward_std": 0.33300668001174927, + "rewards/accuracy_reward/mean": 0.01953125, + "rewards/accuracy_reward/std": 0.1385180652141571, + "rewards/format_reward/mean": 0.0078125, + "rewards/format_reward/std": 0.08812850713729858, + "rewards/tag_count_reward/mean": 0.74462890625, + "rewards/tag_count_reward/std": 0.2910865247249603, + "step": 708 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07421875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 835.6171875, + "completions/mean_terminated_length": 738.421875, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "epoch": 0.24204147819407698, + "grad_norm": 10.483278274536133, + "kl": 4.9921875, + "learning_rate": 9.458571559341849e-07, + "loss": 0.3835, + "num_tokens": 459093640.0, + "reward": 0.8203125, + "reward_std": 0.3339378237724304, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.7734375, + "rewards/tag_count_reward/std": 0.27123603224754333, + "step": 709 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.060546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1990.0, + "completions/mean_length": 740.6640625, + "completions/mean_terminated_length": 656.407470703125, + "completions/min_length": 119.0, + "completions/min_terminated_length": 119.0, + "epoch": 0.2423828625074678, + "grad_norm": 14.420405387878418, + "kl": 4.7890625, + "learning_rate": 9.456019228561425e-07, + "loss": 0.3945, + "num_tokens": 459544044.0, + "reward": 0.90966796875, + "reward_std": 0.3614538908004761, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.80810546875, + "rewards/tag_count_reward/std": 0.26802483201026917, + "step": 710 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.083984375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1987.0, + "completions/mean_length": 838.923828125, + "completions/mean_terminated_length": 728.0703735351562, + "completions/min_length": 107.0, + "completions/min_terminated_length": 107.0, + "epoch": 0.24272424682085858, + "grad_norm": 5.348476409912109, + "kl": 5.1328125, + "learning_rate": 9.453461282935271e-07, + "loss": 0.3805, + "num_tokens": 460050149.0, + "reward": 0.88134765625, + "reward_std": 0.365867555141449, + "rewards/accuracy_reward/mean": 0.09677419066429138, + "rewards/accuracy_reward/std": 0.2959485352039337, + "rewards/format_reward/mean": 0.01171875, + "rewards/format_reward/std": 0.10772226005792618, + "rewards/tag_count_reward/mean": 0.77587890625, + "rewards/tag_count_reward/std": 0.2857325077056885, + "step": 711 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2014.0, + "completions/mean_length": 820.046875, + "completions/mean_terminated_length": 710.3148803710938, + "completions/min_length": 22.0, + "completions/min_terminated_length": 22.0, + "epoch": 0.2430656311342494, + "grad_norm": 6.827632427215576, + "kl": 6.3515625, + "learning_rate": 9.450897726093924e-07, + "loss": 0.4942, + "num_tokens": 460545565.0, + "reward": 0.869140625, + "reward_std": 0.37961816787719727, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.009765625, + "rewards/format_reward/std": 0.09843364357948303, + "rewards/tag_count_reward/mean": 0.76953125, + "rewards/tag_count_reward/std": 0.2898467481136322, + "step": 712 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.09375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2029.0, + "completions/mean_length": 825.546875, + "completions/mean_terminated_length": 699.086181640625, + "completions/min_length": 56.0, + "completions/min_terminated_length": 56.0, + "epoch": 0.24340701544764018, + "grad_norm": 7.4668121337890625, + "kl": 7.359375, + "learning_rate": 9.448328561675883e-07, + "loss": 0.5807, + "num_tokens": 461042709.0, + "reward": 0.8330078125, + "reward_std": 0.32038432359695435, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.7900390625, + "rewards/tag_count_reward/std": 0.2737869620323181, + "step": 713 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0703125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2019.0, + "completions/mean_length": 775.900390625, + "completions/mean_terminated_length": 679.6912231445312, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.243748399761031, + "grad_norm": 7.458494663238525, + "kl": 7.953125, + "learning_rate": 9.44575379332761e-07, + "loss": 0.514, + "num_tokens": 461517026.0, + "reward": 0.8349609375, + "reward_std": 0.33914488554000854, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.013671875, + "rewards/format_reward/std": 0.1162383034825325, + "rewards/tag_count_reward/mean": 0.7724609375, + "rewards/tag_count_reward/std": 0.2810613811016083, + "step": 714 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.083984375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 811.75, + "completions/mean_terminated_length": 698.4051513671875, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.24408978407442178, + "grad_norm": 8.241525650024414, + "kl": 9.359375, + "learning_rate": 9.443173424703514e-07, + "loss": 0.6233, + "num_tokens": 462003106.0, + "reward": 0.8427734375, + "reward_std": 0.35059112310409546, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.0078125, + "rewards/format_reward/std": 0.08812850713729858, + "rewards/tag_count_reward/mean": 0.7646484375, + "rewards/tag_count_reward/std": 0.277199923992157, + "step": 715 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.068359375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2028.0, + "completions/mean_length": 850.09765625, + "completions/mean_terminated_length": 762.2012329101562, + "completions/min_length": 34.0, + "completions/min_terminated_length": 34.0, + "epoch": 0.2444311683878126, + "grad_norm": 14.613713264465332, + "kl": 8.53125, + "learning_rate": 9.440587459465956e-07, + "loss": 0.4895, + "num_tokens": 462516420.0, + "reward": 0.80224609375, + "reward_std": 0.3848583698272705, + "rewards/accuracy_reward/mean": 0.04233871027827263, + "rewards/accuracy_reward/std": 0.2015640139579773, + "rewards/format_reward/mean": 0.021484375, + "rewards/format_reward/std": 0.14513419568538666, + "rewards/tag_count_reward/mean": 0.73974609375, + "rewards/tag_count_reward/std": 0.2884219288825989, + "step": 716 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 868.58203125, + "completions/mean_terminated_length": 784.6903686523438, + "completions/min_length": 54.0, + "completions/min_terminated_length": 54.0, + "epoch": 0.24477255270120338, + "grad_norm": 13.657511711120605, + "kl": 9.0859375, + "learning_rate": 9.437995901285246e-07, + "loss": 0.5348, + "num_tokens": 463034974.0, + "reward": 0.8486328125, + "reward_std": 0.3591812252998352, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.7724609375, + "rewards/tag_count_reward/std": 0.2726678252220154, + "step": 717 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1948.0, + "completions/mean_length": 801.046875, + "completions/mean_terminated_length": 689.6170043945312, + "completions/min_length": 69.0, + "completions/min_terminated_length": 69.0, + "epoch": 0.2451139370145942, + "grad_norm": 7.152915000915527, + "kl": 8.328125, + "learning_rate": 9.435398753839622e-07, + "loss": 0.5683, + "num_tokens": 463519878.0, + "reward": 0.8486328125, + "reward_std": 0.36477065086364746, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.015625, + "rewards/format_reward/std": 0.12414088100194931, + "rewards/tag_count_reward/mean": 0.7724609375, + "rewards/tag_count_reward/std": 0.2722189128398895, + "step": 718 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 802.802734375, + "completions/mean_terminated_length": 719.7896118164062, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.24545532132798498, + "grad_norm": 2.03483510017395, + "kl": 7.25, + "learning_rate": 9.432796020815261e-07, + "loss": 0.5188, + "num_tokens": 464007617.0, + "reward": 0.826171875, + "reward_std": 0.33768337965011597, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.7734375, + "rewards/tag_count_reward/std": 0.2844424843788147, + "step": 719 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2030.0, + "completions/mean_length": 758.4921875, + "completions/mean_terminated_length": 695.07373046875, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.2457967056413758, + "grad_norm": 5.960138320922852, + "kl": 5.21875, + "learning_rate": 9.430187705906268e-07, + "loss": 0.3848, + "num_tokens": 464465917.0, + "reward": 0.86767578125, + "reward_std": 0.3609304130077362, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.013671875, + "rewards/format_reward/std": 0.1162383034825325, + "rewards/tag_count_reward/mean": 0.78369140625, + "rewards/tag_count_reward/std": 0.2712813913822174, + "step": 720 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1987.0, + "completions/mean_length": 803.236328125, + "completions/mean_terminated_length": 739.3367919921875, + "completions/min_length": 24.0, + "completions/min_terminated_length": 24.0, + "epoch": 0.24613808995476658, + "grad_norm": 7.317914009094238, + "kl": 4.87890625, + "learning_rate": 9.427573812814666e-07, + "loss": 0.3755, + "num_tokens": 464947862.0, + "reward": 0.89453125, + "reward_std": 0.3675106167793274, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.791015625, + "rewards/tag_count_reward/std": 0.27139803767204285, + "step": 721 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1968.0, + "completions/mean_length": 773.197265625, + "completions/mean_terminated_length": 702.2288818359375, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.2464794742681574, + "grad_norm": 7.03244686126709, + "kl": 4.75390625, + "learning_rate": 9.424954345250401e-07, + "loss": 0.3533, + "num_tokens": 465410731.0, + "reward": 0.83984375, + "reward_std": 0.29207414388656616, + "rewards/accuracy_reward/mean": 0.021484375, + "rewards/accuracy_reward/std": 0.14513419568538666, + "rewards/format_reward/mean": 0.009765625, + "rewards/format_reward/std": 0.09843364357948303, + "rewards/tag_count_reward/mean": 0.80859375, + "rewards/tag_count_reward/std": 0.2630811929702759, + "step": 722 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 816.486328125, + "completions/mean_terminated_length": 700.7030029296875, + "completions/min_length": 50.0, + "completions/min_terminated_length": 50.0, + "epoch": 0.24682085858154817, + "grad_norm": 5.78585147857666, + "kl": 5.56640625, + "learning_rate": 9.422329306931325e-07, + "loss": 0.4128, + "num_tokens": 465902532.0, + "reward": 0.83154296875, + "reward_std": 0.3125340938568115, + "rewards/accuracy_reward/mean": 0.02734375, + "rewards/accuracy_reward/std": 0.16324250400066376, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.79833984375, + "rewards/tag_count_reward/std": 0.27311384677886963, + "step": 723 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 797.927734375, + "completions/mean_terminated_length": 733.7556762695312, + "completions/min_length": 35.0, + "completions/min_terminated_length": 35.0, + "epoch": 0.247162242894939, + "grad_norm": 2.1381256580352783, + "kl": 6.3203125, + "learning_rate": 9.419698701583204e-07, + "loss": 0.4465, + "num_tokens": 466392495.0, + "reward": 0.8359375, + "reward_std": 0.367983341217041, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.013671875, + "rewards/format_reward/std": 0.1162383034825325, + "rewards/tag_count_reward/mean": 0.767578125, + "rewards/tag_count_reward/std": 0.28400543332099915, + "step": 724 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07421875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2030.0, + "completions/mean_length": 818.634765625, + "completions/mean_terminated_length": 720.0780029296875, + "completions/min_length": 52.0, + "completions/min_terminated_length": 52.0, + "epoch": 0.24750362720832977, + "grad_norm": 4.268383502960205, + "kl": 8.0078125, + "learning_rate": 9.417062532939698e-07, + "loss": 0.5542, + "num_tokens": 466887652.0, + "reward": 0.82861328125, + "reward_std": 0.3431798815727234, + "rewards/accuracy_reward/mean": 0.04032257944345474, + "rewards/accuracy_reward/std": 0.19691328704357147, + "rewards/format_reward/mean": 0.009765625, + "rewards/format_reward/std": 0.09843364357948303, + "rewards/tag_count_reward/mean": 0.77978515625, + "rewards/tag_count_reward/std": 0.2748715281486511, + "step": 725 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.076171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 811.517578125, + "completions/mean_terminated_length": 709.5665893554688, + "completions/min_length": 10.0, + "completions/min_terminated_length": 10.0, + "epoch": 0.2478450115217206, + "grad_norm": 3.5171008110046387, + "kl": 7.375, + "learning_rate": 9.414420804742366e-07, + "loss": 0.4769, + "num_tokens": 467385501.0, + "reward": 0.80908203125, + "reward_std": 0.3296193480491638, + "rewards/accuracy_reward/mean": 0.03125, + "rewards/accuracy_reward/std": 0.17416280508041382, + "rewards/format_reward/mean": 0.015625, + "rewards/format_reward/std": 0.12414088100194931, + "rewards/tag_count_reward/mean": 0.76220703125, + "rewards/tag_count_reward/std": 0.27929604053497314, + "step": 726 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 797.9375, + "completions/mean_terminated_length": 714.6000366210938, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "epoch": 0.24818639583511137, + "grad_norm": 10.195523262023926, + "kl": 6.859375, + "learning_rate": 9.41177352074066e-07, + "loss": 0.4419, + "num_tokens": 467876237.0, + "reward": 0.79736328125, + "reward_std": 0.3004957437515259, + "rewards/accuracy_reward/mean": 0.009765625, + "rewards/accuracy_reward/std": 0.09843364357948303, + "rewards/format_reward/mean": 0.009765625, + "rewards/format_reward/std": 0.09843364357948303, + "rewards/tag_count_reward/mean": 0.77783203125, + "rewards/tag_count_reward/std": 0.26968812942504883, + "step": 727 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 800.2578125, + "completions/mean_terminated_length": 733.5061645507812, + "completions/min_length": 40.0, + "completions/min_terminated_length": 40.0, + "epoch": 0.2485277801485022, + "grad_norm": 3.3900604248046875, + "kl": 6.796875, + "learning_rate": 9.409120684691915e-07, + "loss": 0.4141, + "num_tokens": 468363137.0, + "reward": 0.81640625, + "reward_std": 0.321017324924469, + "rewards/accuracy_reward/mean": 0.032258063554763794, + "rewards/accuracy_reward/std": 0.17686307430267334, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.78125, + "rewards/tag_count_reward/std": 0.26542437076568604, + "step": 728 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.060546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1963.0, + "completions/mean_length": 793.02734375, + "completions/mean_terminated_length": 712.1455078125, + "completions/min_length": 20.0, + "completions/min_terminated_length": 20.0, + "epoch": 0.24886916446189297, + "grad_norm": 2.7198610305786133, + "kl": 6.46875, + "learning_rate": 9.406462300361345e-07, + "loss": 0.4298, + "num_tokens": 468844687.0, + "reward": 0.80908203125, + "reward_std": 0.3425788879394531, + "rewards/accuracy_reward/mean": 0.037109375, + "rewards/accuracy_reward/std": 0.18921469151973724, + "rewards/format_reward/mean": 0.01171875, + "rewards/format_reward/std": 0.10772226005792618, + "rewards/tag_count_reward/mean": 0.76025390625, + "rewards/tag_count_reward/std": 0.2802489399909973, + "step": 729 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1966.0, + "completions/mean_length": 744.380859375, + "completions/mean_terminated_length": 680.2684326171875, + "completions/min_length": 34.0, + "completions/min_terminated_length": 34.0, + "epoch": 0.24921054877528379, + "grad_norm": 1.6548537015914917, + "kl": 5.2265625, + "learning_rate": 9.403798371522042e-07, + "loss": 0.3176, + "num_tokens": 469297794.0, + "reward": 0.88818359375, + "reward_std": 0.3665573298931122, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.021484375, + "rewards/format_reward/std": 0.14513419568538666, + "rewards/tag_count_reward/mean": 0.77685546875, + "rewards/tag_count_reward/std": 0.2661357820034027, + "step": 730 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2020.0, + "completions/mean_length": 795.10546875, + "completions/mean_terminated_length": 738.85302734375, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "epoch": 0.24955193308867457, + "grad_norm": 8.122745513916016, + "kl": 4.5234375, + "learning_rate": 9.401128901954964e-07, + "loss": 0.3331, + "num_tokens": 469783624.0, + "reward": 0.85302734375, + "reward_std": 0.3369097113609314, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.015625, + "rewards/format_reward/std": 0.12414088100194931, + "rewards/tag_count_reward/mean": 0.78662109375, + "rewards/tag_count_reward/std": 0.2699962258338928, + "step": 731 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.056640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2017.0, + "completions/mean_length": 736.365234375, + "completions/mean_terminated_length": 657.6128540039062, + "completions/min_length": 34.0, + "completions/min_terminated_length": 34.0, + "epoch": 0.24989331740206538, + "grad_norm": 3.018124580383301, + "kl": 5.046875, + "learning_rate": 9.398453895448936e-07, + "loss": 0.3587, + "num_tokens": 470232739.0, + "reward": 0.83642578125, + "reward_std": 0.3421552777290344, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.013671875, + "rewards/format_reward/std": 0.1162383034825325, + "rewards/tag_count_reward/mean": 0.78369140625, + "rewards/tag_count_reward/std": 0.26765021681785583, + "step": 732 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 741.564453125, + "completions/mean_terminated_length": 674.4990234375, + "completions/min_length": 9.0, + "completions/min_terminated_length": 9.0, + "epoch": 0.2502347017154562, + "grad_norm": 4.018496036529541, + "kl": 4.15234375, + "learning_rate": 9.395773355800643e-07, + "loss": 0.268, + "num_tokens": 470697764.0, + "reward": 0.87939453125, + "reward_std": 0.352216899394989, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.015625, + "rewards/format_reward/std": 0.12414088100194931, + "rewards/tag_count_reward/mean": 0.80712890625, + "rewards/tag_count_reward/std": 0.2551484704017639, + "step": 733 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2014.0, + "completions/mean_length": 751.60546875, + "completions/mean_terminated_length": 704.3684692382812, + "completions/min_length": 99.0, + "completions/min_terminated_length": 99.0, + "epoch": 0.25057608602884696, + "grad_norm": 2.691394805908203, + "kl": 5.234375, + "learning_rate": 9.393087286814616e-07, + "loss": 0.3312, + "num_tokens": 471164554.0, + "reward": 0.82421875, + "reward_std": 0.31793129444122314, + "rewards/accuracy_reward/mean": 0.02734375, + "rewards/accuracy_reward/std": 0.16324250400066376, + "rewards/format_reward/mean": 0.01171875, + "rewards/format_reward/std": 0.10772226005792618, + "rewards/tag_count_reward/mean": 0.78515625, + "rewards/tag_count_reward/std": 0.2588631808757782, + "step": 734 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 725.771484375, + "completions/mean_terminated_length": 683.1189575195312, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.25091747034223777, + "grad_norm": 3.793814182281494, + "kl": 5.13671875, + "learning_rate": 9.39039569230324e-07, + "loss": 0.347, + "num_tokens": 471614277.0, + "reward": 0.8583984375, + "reward_std": 0.34439292550086975, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.01171875, + "rewards/format_reward/std": 0.10772226005792618, + "rewards/tag_count_reward/mean": 0.7958984375, + "rewards/tag_count_reward/std": 0.2660554349422455, + "step": 735 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1919.0, + "completions/mean_length": 733.33984375, + "completions/mean_terminated_length": 679.8983764648438, + "completions/min_length": 5.0, + "completions/min_terminated_length": 5.0, + "epoch": 0.2512588546556286, + "grad_norm": 2.680269718170166, + "kl": 6.0625, + "learning_rate": 9.387698576086743e-07, + "loss": 0.3465, + "num_tokens": 472067875.0, + "reward": 0.7998046875, + "reward_std": 0.33144643902778625, + "rewards/accuracy_reward/mean": 0.02734375, + "rewards/accuracy_reward/std": 0.16324250400066376, + "rewards/format_reward/mean": 0.009765625, + "rewards/format_reward/std": 0.09843364357948303, + "rewards/tag_count_reward/mean": 0.7626953125, + "rewards/tag_count_reward/std": 0.27641287446022034, + "step": 736 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1886.0, + "completions/mean_length": 656.005859375, + "completions/mean_terminated_length": 611.102783203125, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.2516002389690194, + "grad_norm": 2.185762882232666, + "kl": 5.390625, + "learning_rate": 9.384995941993187e-07, + "loss": 0.3094, + "num_tokens": 472473798.0, + "reward": 0.87548828125, + "reward_std": 0.3545587956905365, + "rewards/accuracy_reward/mean": 0.07258064299821854, + "rewards/accuracy_reward/std": 0.25970885157585144, + "rewards/format_reward/mean": 0.009765625, + "rewards/format_reward/std": 0.09843364357948303, + "rewards/tag_count_reward/mean": 0.79541015625, + "rewards/tag_count_reward/std": 0.25939005613327026, + "step": 737 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2022.0, + "completions/mean_length": 773.515625, + "completions/mean_terminated_length": 721.707275390625, + "completions/min_length": 12.0, + "completions/min_terminated_length": 12.0, + "epoch": 0.25194162328241015, + "grad_norm": 5.866791725158691, + "kl": 6.046875, + "learning_rate": 9.382287793858467e-07, + "loss": 0.334, + "num_tokens": 472942782.0, + "reward": 0.85400390625, + "reward_std": 0.35613566637039185, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.0078125, + "rewards/format_reward/std": 0.08812850713729858, + "rewards/tag_count_reward/mean": 0.77978515625, + "rewards/tag_count_reward/std": 0.26947900652885437, + "step": 738 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 712.3984375, + "completions/mean_terminated_length": 666.529296875, + "completions/min_length": 14.0, + "completions/min_terminated_length": 14.0, + "epoch": 0.25228300759580097, + "grad_norm": 2.493330717086792, + "kl": 5.6796875, + "learning_rate": 9.379574135526304e-07, + "loss": 0.3737, + "num_tokens": 473382570.0, + "reward": 0.8818359375, + "reward_std": 0.36612144112586975, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.013671875, + "rewards/format_reward/std": 0.1162383034825325, + "rewards/tag_count_reward/mean": 0.7822265625, + "rewards/tag_count_reward/std": 0.2666867971420288, + "step": 739 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 695.375, + "completions/mean_terminated_length": 646.089111328125, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "epoch": 0.2526243919091918, + "grad_norm": 3.5300133228302, + "kl": 5.296875, + "learning_rate": 9.376854970848239e-07, + "loss": 0.3468, + "num_tokens": 473816058.0, + "reward": 0.85400390625, + "reward_std": 0.34921133518218994, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.77587890625, + "rewards/tag_count_reward/std": 0.26897501945495605, + "step": 740 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1957.0, + "completions/mean_length": 717.44921875, + "completions/mean_terminated_length": 643.3773193359375, + "completions/min_length": 37.0, + "completions/min_terminated_length": 37.0, + "epoch": 0.2529657762225826, + "grad_norm": 4.015928268432617, + "kl": 5.875, + "learning_rate": 9.374130303683628e-07, + "loss": 0.3583, + "num_tokens": 474258784.0, + "reward": 0.86572265625, + "reward_std": 0.35375192761421204, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.015625, + "rewards/format_reward/std": 0.12414088100194931, + "rewards/tag_count_reward/mean": 0.77783203125, + "rewards/tag_count_reward/std": 0.25998979806900024, + "step": 741 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2029.0, + "completions/mean_length": 741.83203125, + "completions/mean_terminated_length": 683.187744140625, + "completions/min_length": 11.0, + "completions/min_terminated_length": 11.0, + "epoch": 0.25330716053597335, + "grad_norm": 2.450438976287842, + "kl": 4.73828125, + "learning_rate": 9.371400137899642e-07, + "loss": 0.2917, + "num_tokens": 474723482.0, + "reward": 0.80810546875, + "reward_std": 0.33236464858055115, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.013671875, + "rewards/format_reward/std": 0.1162383034825325, + "rewards/tag_count_reward/mean": 0.75341796875, + "rewards/tag_count_reward/std": 0.27513211965560913, + "step": 742 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1963.0, + "completions/mean_length": 681.587890625, + "completions/mean_terminated_length": 651.5868530273438, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "epoch": 0.25364854484936417, + "grad_norm": 5.370906352996826, + "kl": 4.0, + "learning_rate": 9.368664477371246e-07, + "loss": 0.2457, + "num_tokens": 475151095.0, + "reward": 0.84375, + "reward_std": 0.3205603063106537, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.798828125, + "rewards/tag_count_reward/std": 0.2556764483451843, + "step": 743 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2019.0, + "completions/mean_length": 729.072265625, + "completions/mean_terminated_length": 681.01416015625, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.253989929162755, + "grad_norm": 3.3952784538269043, + "kl": 3.8984375, + "learning_rate": 9.365923325981214e-07, + "loss": 0.2532, + "num_tokens": 475597372.0, + "reward": 0.810546875, + "reward_std": 0.31725451350212097, + "rewards/accuracy_reward/mean": 0.01953125, + "rewards/accuracy_reward/std": 0.1385180652141571, + "rewards/format_reward/mean": 0.013671875, + "rewards/format_reward/std": 0.1162383034825325, + "rewards/tag_count_reward/mean": 0.77734375, + "rewards/tag_count_reward/std": 0.26677456498146057, + "step": 744 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1975.0, + "completions/mean_length": 697.048828125, + "completions/mean_terminated_length": 650.6525268554688, + "completions/min_length": 14.0, + "completions/min_terminated_length": 14.0, + "epoch": 0.2543313134761458, + "grad_norm": 2.641296863555908, + "kl": 5.328125, + "learning_rate": 9.363176687620109e-07, + "loss": 0.3497, + "num_tokens": 476029461.0, + "reward": 0.8369140625, + "reward_std": 0.3765317499637604, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.009765625, + "rewards/format_reward/std": 0.09843364357948303, + "rewards/tag_count_reward/mean": 0.7490234375, + "rewards/tag_count_reward/std": 0.268628865480423, + "step": 745 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2000.0, + "completions/mean_length": 715.10546875, + "completions/mean_terminated_length": 655.2611694335938, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "epoch": 0.25467269778953655, + "grad_norm": 2.496892213821411, + "kl": 5.859375, + "learning_rate": 9.360424566186279e-07, + "loss": 0.3758, + "num_tokens": 476473035.0, + "reward": 0.8193359375, + "reward_std": 0.3387095034122467, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.7626953125, + "rewards/tag_count_reward/std": 0.26878535747528076, + "step": 746 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1993.0, + "completions/mean_length": 648.65234375, + "completions/mean_terminated_length": 603.5120849609375, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.25501408210292736, + "grad_norm": 4.981993675231934, + "kl": 5.578125, + "learning_rate": 9.357666965585859e-07, + "loss": 0.3321, + "num_tokens": 476886873.0, + "reward": 0.84375, + "reward_std": 0.35605841875076294, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.021484375, + "rewards/format_reward/std": 0.14513419568538666, + "rewards/tag_count_reward/mean": 0.775390625, + "rewards/tag_count_reward/std": 0.27285876870155334, + "step": 747 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 791.423828125, + "completions/mean_terminated_length": 724.1995849609375, + "completions/min_length": 8.0, + "completions/min_terminated_length": 8.0, + "epoch": 0.2553554664163182, + "grad_norm": 1.5774364471435547, + "kl": 5.890625, + "learning_rate": 9.354903889732761e-07, + "loss": 0.4023, + "num_tokens": 477365858.0, + "reward": 0.8251953125, + "reward_std": 0.3529778718948364, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.013671875, + "rewards/format_reward/std": 0.1162383034825325, + "rewards/tag_count_reward/mean": 0.7646484375, + "rewards/tag_count_reward/std": 0.2645571529865265, + "step": 748 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1902.0, + "completions/mean_length": 682.677734375, + "completions/mean_terminated_length": 661.0059814453125, + "completions/min_length": 14.0, + "completions/min_terminated_length": 14.0, + "epoch": 0.255696850729709, + "grad_norm": 2.7102158069610596, + "kl": 3.68359375, + "learning_rate": 9.352135342548659e-07, + "loss": 0.1737, + "num_tokens": 477801197.0, + "reward": 0.84228515625, + "reward_std": 0.3041990101337433, + "rewards/accuracy_reward/mean": 0.021484375, + "rewards/accuracy_reward/std": 0.14513419568538666, + "rewards/format_reward/mean": 0.01171875, + "rewards/format_reward/std": 0.10772226005792618, + "rewards/tag_count_reward/mean": 0.80908203125, + "rewards/tag_count_reward/std": 0.25034329295158386, + "step": 749 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1904.0, + "completions/mean_length": 707.30859375, + "completions/mean_terminated_length": 652.8088989257812, + "completions/min_length": 13.0, + "completions/min_terminated_length": 13.0, + "epoch": 0.25603823504309975, + "grad_norm": 6.120353698730469, + "kl": 5.9296875, + "learning_rate": 9.349361327963006e-07, + "loss": 0.3421, + "num_tokens": 478238651.0, + "reward": 0.84130859375, + "reward_std": 0.33035802841186523, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.77490234375, + "rewards/tag_count_reward/std": 0.27223074436187744, + "step": 750 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 664.611328125, + "completions/mean_terminated_length": 639.8588256835938, + "completions/min_length": 18.0, + "completions/min_terminated_length": 18.0, + "epoch": 0.25637961935649056, + "grad_norm": 1.5641052722930908, + "kl": 3.7265625, + "learning_rate": 9.346581849913004e-07, + "loss": 0.1939, + "num_tokens": 478658340.0, + "reward": 0.91552734375, + "reward_std": 0.3789862394332886, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.0078125, + "rewards/format_reward/std": 0.08812850713729858, + "rewards/tag_count_reward/mean": 0.81787109375, + "rewards/tag_count_reward/std": 0.2553880512714386, + "step": 751 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1995.0, + "completions/mean_length": 725.490234375, + "completions/mean_terminated_length": 685.575439453125, + "completions/min_length": 27.0, + "completions/min_terminated_length": 27.0, + "epoch": 0.2567210036698814, + "grad_norm": 1.9023103713989258, + "kl": 4.8203125, + "learning_rate": 9.343796912343617e-07, + "loss": 0.2562, + "num_tokens": 479109839.0, + "reward": 0.83349609375, + "reward_std": 0.3289315700531006, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.78271484375, + "rewards/tag_count_reward/std": 0.264554888010025, + "step": 752 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1998.0, + "completions/mean_length": 719.732421875, + "completions/mean_terminated_length": 682.3915405273438, + "completions/min_length": 18.0, + "completions/min_terminated_length": 18.0, + "epoch": 0.2570623879832722, + "grad_norm": 4.725130081176758, + "kl": 5.1640625, + "learning_rate": 9.341006519207551e-07, + "loss": 0.2603, + "num_tokens": 479555462.0, + "reward": 0.84033203125, + "reward_std": 0.33210453391075134, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.01171875, + "rewards/format_reward/std": 0.10772226005792618, + "rewards/tag_count_reward/mean": 0.78369140625, + "rewards/tag_count_reward/std": 0.25549277663230896, + "step": 753 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2031.0, + "completions/mean_length": 731.30859375, + "completions/mean_terminated_length": 697.0060424804688, + "completions/min_length": 99.0, + "completions/min_terminated_length": 99.0, + "epoch": 0.25740377229666295, + "grad_norm": 3.288994550704956, + "kl": 3.8671875, + "learning_rate": 9.338210674465263e-07, + "loss": 0.2331, + "num_tokens": 480010244.0, + "reward": 0.861328125, + "reward_std": 0.31858742237091064, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.009765625, + "rewards/format_reward/std": 0.09843364357948303, + "rewards/tag_count_reward/mean": 0.8046875, + "rewards/tag_count_reward/std": 0.2491423636674881, + "step": 754 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1994.0, + "completions/mean_length": 694.751953125, + "completions/mean_terminated_length": 675.9940795898438, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.25774515661005376, + "grad_norm": 2.6497554779052734, + "kl": 4.734375, + "learning_rate": 9.335409382084939e-07, + "loss": 0.2692, + "num_tokens": 480439381.0, + "reward": 0.87548828125, + "reward_std": 0.32045841217041016, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.80517578125, + "rewards/tag_count_reward/std": 0.24878878891468048, + "step": 755 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2023.0, + "completions/mean_length": 720.1015625, + "completions/mean_terminated_length": 666.1219482421875, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.2580865409234446, + "grad_norm": 4.2875871658325195, + "kl": 5.5234375, + "learning_rate": 9.332602646042504e-07, + "loss": 0.3288, + "num_tokens": 480884121.0, + "reward": 0.87646484375, + "reward_std": 0.3499682545661926, + "rewards/accuracy_reward/mean": 0.07459677755832672, + "rewards/accuracy_reward/std": 0.263004869222641, + "rewards/format_reward/mean": 0.0078125, + "rewards/format_reward/std": 0.08812850713729858, + "rewards/tag_count_reward/mean": 0.79638671875, + "rewards/tag_count_reward/std": 0.2582714557647705, + "step": 756 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2006.0, + "completions/mean_length": 739.109375, + "completions/mean_terminated_length": 720.9663696289062, + "completions/min_length": 14.0, + "completions/min_terminated_length": 14.0, + "epoch": 0.2584279252368354, + "grad_norm": 1.863918423652649, + "kl": 3.87890625, + "learning_rate": 9.329790470321607e-07, + "loss": 0.1861, + "num_tokens": 481341153.0, + "reward": 0.84033203125, + "reward_std": 0.3457339107990265, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.0078125, + "rewards/format_reward/std": 0.08812850713729858, + "rewards/tag_count_reward/mean": 0.78564453125, + "rewards/tag_count_reward/std": 0.26876533031463623, + "step": 757 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1997.0, + "completions/mean_length": 766.458984375, + "completions/mean_terminated_length": 743.52880859375, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "epoch": 0.25876930955022615, + "grad_norm": 1.599083423614502, + "kl": 4.2109375, + "learning_rate": 9.326972858913613e-07, + "loss": 0.2422, + "num_tokens": 481813196.0, + "reward": 0.89599609375, + "reward_std": 0.3567178547382355, + "rewards/accuracy_reward/mean": 0.08266129344701767, + "rewards/accuracy_reward/std": 0.2756476104259491, + "rewards/format_reward/mean": 0.01171875, + "rewards/format_reward/std": 0.10772226005792618, + "rewards/tag_count_reward/mean": 0.80419921875, + "rewards/tag_count_reward/std": 0.24949447810649872, + "step": 758 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.009765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1809.0, + "completions/mean_length": 669.76953125, + "completions/mean_terminated_length": 656.1775512695312, + "completions/min_length": 16.0, + "completions/min_terminated_length": 16.0, + "epoch": 0.25911069386361696, + "grad_norm": 2.088303565979004, + "kl": 4.078125, + "learning_rate": 9.324149815817612e-07, + "loss": 0.1918, + "num_tokens": 482228918.0, + "reward": 0.888671875, + "reward_std": 0.3486194908618927, + "rewards/accuracy_reward/mean": 0.07258064299821854, + "rewards/accuracy_reward/std": 0.25970885157585144, + "rewards/format_reward/mean": 0.01171875, + "rewards/format_reward/std": 0.10772226005792618, + "rewards/tag_count_reward/mean": 0.806640625, + "rewards/tag_count_reward/std": 0.25740471482276917, + "step": 759 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1974.0, + "completions/mean_length": 714.9921875, + "completions/mean_terminated_length": 683.0000610351562, + "completions/min_length": 17.0, + "completions/min_terminated_length": 17.0, + "epoch": 0.25945207817700777, + "grad_norm": 1.6705163717269897, + "kl": 4.390625, + "learning_rate": 9.321321345040391e-07, + "loss": 0.2625, + "num_tokens": 482675394.0, + "reward": 0.84375, + "reward_std": 0.3496783375740051, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.0078125, + "rewards/format_reward/std": 0.08812850713729858, + "rewards/tag_count_reward/mean": 0.783203125, + "rewards/tag_count_reward/std": 0.26240473985671997, + "step": 760 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1916.0, + "completions/mean_length": 723.802734375, + "completions/mean_terminated_length": 702.7837524414062, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.2597934624903986, + "grad_norm": 6.62919807434082, + "kl": 3.78515625, + "learning_rate": 9.31848745059645e-07, + "loss": 0.2394, + "num_tokens": 483122077.0, + "reward": 0.892578125, + "reward_std": 0.3438825309276581, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.009765625, + "rewards/format_reward/std": 0.09843364357948303, + "rewards/tag_count_reward/mean": 0.798828125, + "rewards/tag_count_reward/std": 0.2551976144313812, + "step": 761 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2002.0, + "completions/mean_length": 713.390625, + "completions/mean_terminated_length": 686.8048095703125, + "completions/min_length": 21.0, + "completions/min_terminated_length": 21.0, + "epoch": 0.26013484680378934, + "grad_norm": 7.827740669250488, + "kl": 4.15234375, + "learning_rate": 9.315648136507987e-07, + "loss": 0.2751, + "num_tokens": 483560965.0, + "reward": 0.81103515625, + "reward_std": 0.3110986053943634, + "rewards/accuracy_reward/mean": 0.017578125, + "rewards/accuracy_reward/std": 0.13154059648513794, + "rewards/format_reward/mean": 0.009765625, + "rewards/format_reward/std": 0.09843364357948303, + "rewards/tag_count_reward/mean": 0.78369140625, + "rewards/tag_count_reward/std": 0.26443204283714294, + "step": 762 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2007.0, + "completions/mean_length": 718.63671875, + "completions/mean_terminated_length": 670.1984252929688, + "completions/min_length": 22.0, + "completions/min_terminated_length": 22.0, + "epoch": 0.26047623111718016, + "grad_norm": 3.6159653663635254, + "kl": 5.1484375, + "learning_rate": 9.312803406804882e-07, + "loss": 0.2669, + "num_tokens": 484009371.0, + "reward": 0.833984375, + "reward_std": 0.33764201402664185, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.01171875, + "rewards/format_reward/std": 0.10772226005792618, + "rewards/tag_count_reward/mean": 0.771484375, + "rewards/tag_count_reward/std": 0.2691352665424347, + "step": 763 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2031.0, + "completions/mean_length": 700.111328125, + "completions/mean_terminated_length": 675.9940185546875, + "completions/min_length": 9.0, + "completions/min_terminated_length": 9.0, + "epoch": 0.26081761543057097, + "grad_norm": 2.015320301055908, + "kl": 4.87109375, + "learning_rate": 9.309953265524714e-07, + "loss": 0.2861, + "num_tokens": 484439188.0, + "reward": 0.84326171875, + "reward_std": 0.3085383474826813, + "rewards/accuracy_reward/mean": 0.025390625, + "rewards/accuracy_reward/std": 0.15746226906776428, + "rewards/format_reward/mean": 0.015625, + "rewards/format_reward/std": 0.12414088100194931, + "rewards/tag_count_reward/mean": 0.80224609375, + "rewards/tag_count_reward/std": 0.25619494915008545, + "step": 764 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2010.0, + "completions/mean_length": 726.439453125, + "completions/mean_terminated_length": 716.033447265625, + "completions/min_length": 11.0, + "completions/min_terminated_length": 11.0, + "epoch": 0.2611589997439618, + "grad_norm": 2.472776174545288, + "kl": 4.51953125, + "learning_rate": 9.307097716712735e-07, + "loss": 0.2319, + "num_tokens": 484882725.0, + "reward": 0.822265625, + "reward_std": 0.2942245602607727, + "rewards/accuracy_reward/mean": 0.029296875, + "rewards/accuracy_reward/std": 0.16880230605602264, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.791015625, + "rewards/tag_count_reward/std": 0.25847160816192627, + "step": 765 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1962.0, + "completions/mean_length": 738.49609375, + "completions/mean_terminated_length": 707.0680541992188, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "epoch": 0.26150038405735254, + "grad_norm": 1.8773621320724487, + "kl": 5.1875, + "learning_rate": 9.304236764421876e-07, + "loss": 0.3341, + "num_tokens": 485344787.0, + "reward": 0.8388671875, + "reward_std": 0.36534199118614197, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.7646484375, + "rewards/tag_count_reward/std": 0.27852046489715576, + "step": 766 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2030.0, + "completions/mean_length": 719.046875, + "completions/mean_terminated_length": 689.8682861328125, + "completions/min_length": 31.0, + "completions/min_terminated_length": 31.0, + "epoch": 0.26184176837074336, + "grad_norm": 3.4017436504364014, + "kl": 4.6015625, + "learning_rate": 9.301370412712733e-07, + "loss": 0.2944, + "num_tokens": 485790027.0, + "reward": 0.88427734375, + "reward_std": 0.32861924171447754, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.81005859375, + "rewards/tag_count_reward/std": 0.25399237871170044, + "step": 767 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2015.0, + "completions/mean_length": 728.86328125, + "completions/mean_terminated_length": 707.9246215820312, + "completions/min_length": 139.0, + "completions/min_terminated_length": 139.0, + "epoch": 0.26218315268413417, + "grad_norm": 5.394140720367432, + "kl": 4.59375, + "learning_rate": 9.29849866565357e-07, + "loss": 0.3168, + "num_tokens": 486238709.0, + "reward": 0.8681640625, + "reward_std": 0.3399982750415802, + "rewards/accuracy_reward/mean": 0.060483869165182114, + "rewards/accuracy_reward/std": 0.2386218160390854, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.8056640625, + "rewards/tag_count_reward/std": 0.2576180696487427, + "step": 768 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1964.0, + "completions/mean_length": 689.662109375, + "completions/mean_terminated_length": 668.1012573242188, + "completions/min_length": 47.0, + "completions/min_terminated_length": 47.0, + "epoch": 0.262524536997525, + "grad_norm": 14.306845664978027, + "kl": 6.515625, + "learning_rate": 9.295621527320305e-07, + "loss": 0.3435, + "num_tokens": 486661656.0, + "reward": 0.82373046875, + "reward_std": 0.3080504536628723, + "rewards/accuracy_reward/mean": 0.03125, + "rewards/accuracy_reward/std": 0.17416280508041382, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.78857421875, + "rewards/tag_count_reward/std": 0.26329857110977173, + "step": 769 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1986.0, + "completions/mean_length": 808.90234375, + "completions/mean_terminated_length": 761.1480712890625, + "completions/min_length": 31.0, + "completions/min_terminated_length": 31.0, + "epoch": 0.26286592131091574, + "grad_norm": 12.381365776062012, + "kl": 7.7734375, + "learning_rate": 9.292739001796513e-07, + "loss": 0.4548, + "num_tokens": 487157414.0, + "reward": 0.82861328125, + "reward_std": 0.33342158794403076, + "rewards/accuracy_reward/mean": 0.04233871027827263, + "rewards/accuracy_reward/std": 0.2015640139579773, + "rewards/format_reward/mean": 0.0078125, + "rewards/format_reward/std": 0.08812850713729858, + "rewards/tag_count_reward/mean": 0.77978515625, + "rewards/tag_count_reward/std": 0.2685697376728058, + "step": 770 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2017.0, + "completions/mean_length": 741.685546875, + "completions/mean_terminated_length": 710.3340454101562, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "epoch": 0.26320730562430655, + "grad_norm": 7.73433256149292, + "kl": 7.0625, + "learning_rate": 9.289851093173408e-07, + "loss": 0.4275, + "num_tokens": 487621301.0, + "reward": 0.86376953125, + "reward_std": 0.34932851791381836, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.77783203125, + "rewards/tag_count_reward/std": 0.27014127373695374, + "step": 771 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1960.0, + "completions/mean_length": 734.646484375, + "completions/mean_terminated_length": 689.5414428710938, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "epoch": 0.26354868993769737, + "grad_norm": 1.9583275318145752, + "kl": 5.8359375, + "learning_rate": 9.286957805549849e-07, + "loss": 0.3714, + "num_tokens": 488070240.0, + "reward": 0.84033203125, + "reward_std": 0.3109779357910156, + "rewards/accuracy_reward/mean": 0.037109375, + "rewards/accuracy_reward/std": 0.18921469151973724, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.80126953125, + "rewards/tag_count_reward/std": 0.26018086075782776, + "step": 772 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 741.8125, + "completions/mean_terminated_length": 705.0923461914062, + "completions/min_length": 84.0, + "completions/min_terminated_length": 84.0, + "epoch": 0.2638900742510882, + "grad_norm": 5.635120868682861, + "kl": 4.5234375, + "learning_rate": 9.284059143032329e-07, + "loss": 0.3181, + "num_tokens": 488524656.0, + "reward": 0.83935546875, + "reward_std": 0.28641611337661743, + "rewards/accuracy_reward/mean": 0.029296875, + "rewards/accuracy_reward/std": 0.16880230605602264, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.81005859375, + "rewards/tag_count_reward/std": 0.25351038575172424, + "step": 773 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1923.0, + "completions/mean_length": 771.77734375, + "completions/mean_terminated_length": 714.4775390625, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "epoch": 0.26423145856447894, + "grad_norm": 4.567211627960205, + "kl": 4.578125, + "learning_rate": 9.281155109734971e-07, + "loss": 0.37, + "num_tokens": 488997774.0, + "reward": 0.908203125, + "reward_std": 0.3757338225841522, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.0078125, + "rewards/format_reward/std": 0.08812850713729858, + "rewards/tag_count_reward/mean": 0.806640625, + "rewards/tag_count_reward/std": 0.2635094225406647, + "step": 774 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1940.0, + "completions/mean_length": 739.1953125, + "completions/mean_terminated_length": 723.6759033203125, + "completions/min_length": 158.0, + "completions/min_terminated_length": 158.0, + "epoch": 0.26457284287786975, + "grad_norm": 4.73957633972168, + "kl": 4.078125, + "learning_rate": 9.278245709779515e-07, + "loss": 0.2605, + "num_tokens": 489447586.0, + "reward": 0.9208984375, + "reward_std": 0.3388001322746277, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8232421875, + "rewards/tag_count_reward/std": 0.24331942200660706, + "step": 775 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 765.951171875, + "completions/mean_terminated_length": 713.8353271484375, + "completions/min_length": 101.0, + "completions/min_terminated_length": 101.0, + "epoch": 0.26491422719126057, + "grad_norm": 2.601271390914917, + "kl": 5.890625, + "learning_rate": 9.275330947295326e-07, + "loss": 0.4168, + "num_tokens": 489914457.0, + "reward": 0.85888671875, + "reward_std": 0.2817559242248535, + "rewards/accuracy_reward/mean": 0.025390625, + "rewards/accuracy_reward/std": 0.15746226906776428, + "rewards/format_reward/mean": 0.0078125, + "rewards/format_reward/std": 0.08812850713729858, + "rewards/tag_count_reward/mean": 0.82568359375, + "rewards/tag_count_reward/std": 0.2361827939748764, + "step": 776 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2017.0, + "completions/mean_length": 741.03515625, + "completions/mean_terminated_length": 709.6680297851562, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "epoch": 0.2652556115046514, + "grad_norm": 6.369903564453125, + "kl": 7.1171875, + "learning_rate": 9.272410826419374e-07, + "loss": 0.4342, + "num_tokens": 490373355.0, + "reward": 0.8427734375, + "reward_std": 0.29890674352645874, + "rewards/accuracy_reward/mean": 0.021484375, + "rewards/accuracy_reward/std": 0.14513419568538666, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.8173828125, + "rewards/tag_count_reward/std": 0.2562345564365387, + "step": 777 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2005.0, + "completions/mean_length": 734.990234375, + "completions/mean_terminated_length": 706.1616821289062, + "completions/min_length": 10.0, + "completions/min_terminated_length": 10.0, + "epoch": 0.26559699581804214, + "grad_norm": 11.346620559692383, + "kl": 7.3046875, + "learning_rate": 9.269485351296239e-07, + "loss": 0.3906, + "num_tokens": 490827590.0, + "reward": 0.80712890625, + "reward_std": 0.29944953322410583, + "rewards/accuracy_reward/mean": 0.021484375, + "rewards/accuracy_reward/std": 0.14513419568538666, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.78564453125, + "rewards/tag_count_reward/std": 0.26830989122390747, + "step": 778 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1987.0, + "completions/mean_length": 719.546875, + "completions/mean_terminated_length": 679.4526977539062, + "completions/min_length": 21.0, + "completions/min_terminated_length": 21.0, + "epoch": 0.26593838013143295, + "grad_norm": 9.743428230285645, + "kl": 7.1875, + "learning_rate": 9.266554526078095e-07, + "loss": 0.4057, + "num_tokens": 491274734.0, + "reward": 0.92529296875, + "reward_std": 0.3917638063430786, + "rewards/accuracy_reward/mean": 0.107421875, + "rewards/accuracy_reward/std": 0.30995169281959534, + "rewards/format_reward/mean": 0.009765625, + "rewards/format_reward/std": 0.09843364357948303, + "rewards/tag_count_reward/mean": 0.80810546875, + "rewards/tag_count_reward/std": 0.2510598301887512, + "step": 779 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1753.0, + "completions/mean_length": 673.9921875, + "completions/mean_terminated_length": 649.4075317382812, + "completions/min_length": 35.0, + "completions/min_terminated_length": 35.0, + "epoch": 0.26627976444482376, + "grad_norm": 3.9403653144836426, + "kl": 5.1328125, + "learning_rate": 9.263618354924714e-07, + "loss": 0.275, + "num_tokens": 491692586.0, + "reward": 0.8759765625, + "reward_std": 0.2943779528141022, + "rewards/accuracy_reward/mean": 0.04032257944345474, + "rewards/accuracy_reward/std": 0.19691328704357147, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.8349609375, + "rewards/tag_count_reward/std": 0.2504563331604004, + "step": 780 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1956.0, + "completions/mean_length": 742.46484375, + "completions/mean_terminated_length": 716.4581909179688, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "epoch": 0.2666211487582146, + "grad_norm": 3.920292854309082, + "kl": 4.37890625, + "learning_rate": 9.260676842003453e-07, + "loss": 0.2484, + "num_tokens": 492146296.0, + "reward": 0.9189453125, + "reward_std": 0.314945787191391, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.0078125, + "rewards/format_reward/std": 0.08812850713729858, + "rewards/tag_count_reward/mean": 0.8330078125, + "rewards/tag_count_reward/std": 0.2421857714653015, + "step": 781 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2020.0, + "completions/mean_length": 750.123046875, + "completions/mean_terminated_length": 702.83203125, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "epoch": 0.26696253307160533, + "grad_norm": 8.555292129516602, + "kl": 3.08984375, + "learning_rate": 9.257729991489252e-07, + "loss": 0.2254, + "num_tokens": 492605319.0, + "reward": 0.9150390625, + "reward_std": 0.3154516816139221, + "rewards/accuracy_reward/mean": 0.06854838877916336, + "rewards/accuracy_reward/std": 0.25293970108032227, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.8447265625, + "rewards/tag_count_reward/std": 0.2352529615163803, + "step": 782 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1943.0, + "completions/mean_length": 778.810546875, + "completions/mean_terminated_length": 748.3500366210938, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "epoch": 0.26730391738499615, + "grad_norm": 13.338995933532715, + "kl": 3.14453125, + "learning_rate": 9.254777807564626e-07, + "loss": 0.2634, + "num_tokens": 493078374.0, + "reward": 0.88623046875, + "reward_std": 0.3087897002696991, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.83544921875, + "rewards/tag_count_reward/std": 0.24808107316493988, + "step": 783 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 717.474609375, + "completions/mean_terminated_length": 674.554443359375, + "completions/min_length": 160.0, + "completions/min_terminated_length": 160.0, + "epoch": 0.26764530169838696, + "grad_norm": 11.409294128417969, + "kl": 2.8984375, + "learning_rate": 9.251820294419661e-07, + "loss": 0.2482, + "num_tokens": 493520121.0, + "reward": 0.9228515625, + "reward_std": 0.33445990085601807, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.8447265625, + "rewards/tag_count_reward/std": 0.23421084880828857, + "step": 784 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1972.0, + "completions/mean_length": 707.994140625, + "completions/mean_terminated_length": 678.5728759765625, + "completions/min_length": 84.0, + "completions/min_terminated_length": 84.0, + "epoch": 0.2679866860117778, + "grad_norm": 8.545931816101074, + "kl": 2.40234375, + "learning_rate": 9.248857456252005e-07, + "loss": 0.1561, + "num_tokens": 493955734.0, + "reward": 0.935546875, + "reward_std": 0.26695406436920166, + "rewards/accuracy_reward/mean": 0.08870967477560043, + "rewards/accuracy_reward/std": 0.284611314535141, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.84765625, + "rewards/tag_count_reward/std": 0.23036254942417145, + "step": 785 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2019.0, + "completions/mean_length": 773.412109375, + "completions/mean_terminated_length": 740.2064208984375, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "epoch": 0.26832807032516853, + "grad_norm": 2.3647966384887695, + "kl": 3.6171875, + "learning_rate": 9.245889297266866e-07, + "loss": 0.2264, + "num_tokens": 494425017.0, + "reward": 0.90771484375, + "reward_std": 0.319466769695282, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.84130859375, + "rewards/tag_count_reward/std": 0.23216763138771057, + "step": 786 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2030.0, + "completions/mean_length": 779.95703125, + "completions/mean_terminated_length": 717.59423828125, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "epoch": 0.26866945463855935, + "grad_norm": 4.059122085571289, + "kl": 6.2890625, + "learning_rate": 9.242915821677001e-07, + "loss": 0.3614, + "num_tokens": 494898771.0, + "reward": 0.89306640625, + "reward_std": 0.33708655834198, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.80712890625, + "rewards/tag_count_reward/std": 0.25942689180374146, + "step": 787 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1951.0, + "completions/mean_length": 680.892578125, + "completions/mean_terminated_length": 675.5314331054688, + "completions/min_length": 35.0, + "completions/min_terminated_length": 35.0, + "epoch": 0.26901083895195016, + "grad_norm": 3.4603018760681152, + "kl": 4.7109375, + "learning_rate": 9.239937033702717e-07, + "loss": 0.2414, + "num_tokens": 495322892.0, + "reward": 0.9091796875, + "reward_std": 0.3355030119419098, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.8310546875, + "rewards/tag_count_reward/std": 0.25464847683906555, + "step": 788 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1991.0, + "completions/mean_length": 695.173828125, + "completions/mean_terminated_length": 662.7060546875, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "epoch": 0.269352223265341, + "grad_norm": 3.6373889446258545, + "kl": 5.390625, + "learning_rate": 9.236952937571856e-07, + "loss": 0.3142, + "num_tokens": 495755109.0, + "reward": 0.95166015625, + "reward_std": 0.3479437530040741, + "rewards/accuracy_reward/mean": 0.115234375, + "rewards/accuracy_reward/std": 0.3196168541908264, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.83251953125, + "rewards/tag_count_reward/std": 0.2490728348493576, + "step": 789 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2024.0, + "completions/mean_length": 745.138671875, + "completions/mean_terminated_length": 721.8270263671875, + "completions/min_length": 70.0, + "completions/min_terminated_length": 70.0, + "epoch": 0.26969360757873173, + "grad_norm": 7.922231674194336, + "kl": 4.96875, + "learning_rate": 9.233963537519799e-07, + "loss": 0.2474, + "num_tokens": 496211196.0, + "reward": 0.947265625, + "reward_std": 0.3379775583744049, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.845703125, + "rewards/tag_count_reward/std": 0.23949779570102692, + "step": 790 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1910.0, + "completions/mean_length": 661.783203125, + "completions/mean_terminated_length": 642.568359375, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "epoch": 0.27003499189212254, + "grad_norm": 3.0672128200531006, + "kl": 3.75390625, + "learning_rate": 9.230968837789451e-07, + "loss": 0.2257, + "num_tokens": 496615261.0, + "reward": 0.90283203125, + "reward_std": 0.29471856355667114, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.84033203125, + "rewards/tag_count_reward/std": 0.24284131824970245, + "step": 791 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1746.0, + "completions/mean_length": 734.173828125, + "completions/mean_terminated_length": 686.3016357421875, + "completions/min_length": 19.0, + "completions/min_terminated_length": 19.0, + "epoch": 0.27037637620551336, + "grad_norm": 1.4163967370986938, + "kl": 4.34765625, + "learning_rate": 9.227968842631243e-07, + "loss": 0.2629, + "num_tokens": 497067414.0, + "reward": 0.865234375, + "reward_std": 0.2757364511489868, + "rewards/accuracy_reward/mean": 0.021484375, + "rewards/accuracy_reward/std": 0.14513419568538666, + "rewards/format_reward/mean": 0.0078125, + "rewards/format_reward/std": 0.08812850713729858, + "rewards/tag_count_reward/mean": 0.8359375, + "rewards/tag_count_reward/std": 0.24014326930046082, + "step": 792 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 757.34375, + "completions/mean_terminated_length": 731.6334838867188, + "completions/min_length": 43.0, + "completions/min_terminated_length": 43.0, + "epoch": 0.27071776051890417, + "grad_norm": 4.963979244232178, + "kl": 3.953125, + "learning_rate": 9.224963556303116e-07, + "loss": 0.2585, + "num_tokens": 497534902.0, + "reward": 0.9013671875, + "reward_std": 0.3063977062702179, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.8408203125, + "rewards/tag_count_reward/std": 0.23627431690692902, + "step": 793 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1953.0, + "completions/mean_length": 717.626953125, + "completions/mean_terminated_length": 701.851806640625, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.27105914483229493, + "grad_norm": 2.173969030380249, + "kl": 3.97265625, + "learning_rate": 9.221952983070526e-07, + "loss": 0.2175, + "num_tokens": 497992423.0, + "reward": 0.8916015625, + "reward_std": 0.28696465492248535, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.8427734375, + "rewards/tag_count_reward/std": 0.23603153228759766, + "step": 794 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2002.0, + "completions/mean_length": 729.138671875, + "completions/mean_terminated_length": 702.8665771484375, + "completions/min_length": 9.0, + "completions/min_terminated_length": 9.0, + "epoch": 0.27140052914568574, + "grad_norm": 2.111781358718872, + "kl": 4.3125, + "learning_rate": 9.218937127206432e-07, + "loss": 0.2517, + "num_tokens": 498433902.0, + "reward": 0.94580078125, + "reward_std": 0.3448649048805237, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.85009765625, + "rewards/tag_count_reward/std": 0.232225239276886, + "step": 795 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1945.0, + "completions/mean_length": 734.720703125, + "completions/mean_terminated_length": 708.5597534179688, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "epoch": 0.27174191345907656, + "grad_norm": 1.4413419961929321, + "kl": 3.984375, + "learning_rate": 9.215915992991289e-07, + "loss": 0.2238, + "num_tokens": 498886735.0, + "reward": 0.88037109375, + "reward_std": 0.2774122357368469, + "rewards/accuracy_reward/mean": 0.033203125, + "rewards/accuracy_reward/std": 0.17934183776378632, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.84326171875, + "rewards/tag_count_reward/std": 0.22441160678863525, + "step": 796 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1881.0, + "completions/mean_length": 697.185546875, + "completions/mean_terminated_length": 678.46142578125, + "completions/min_length": 127.0, + "completions/min_terminated_length": 127.0, + "epoch": 0.27208329777246737, + "grad_norm": 4.815097332000732, + "kl": 3.9296875, + "learning_rate": 9.212889584713044e-07, + "loss": 0.2891, + "num_tokens": 499322286.0, + "reward": 1.0068359375, + "reward_std": 0.3376759886741638, + "rewards/accuracy_reward/mean": 0.130859375, + "rewards/accuracy_reward/std": 0.33757632970809937, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.8701171875, + "rewards/tag_count_reward/std": 0.2233344167470932, + "step": 797 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1962.0, + "completions/mean_length": 746.00390625, + "completions/mean_terminated_length": 695.8255615234375, + "completions/min_length": 108.0, + "completions/min_terminated_length": 108.0, + "epoch": 0.2724246820858581, + "grad_norm": 2.2542262077331543, + "kl": 5.3984375, + "learning_rate": 9.20985790666713e-07, + "loss": 0.353, + "num_tokens": 499781520.0, + "reward": 0.91015625, + "reward_std": 0.31544923782348633, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.84375, + "rewards/tag_count_reward/std": 0.24975526332855225, + "step": 798 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 790.234375, + "completions/mean_terminated_length": 736.43994140625, + "completions/min_length": 32.0, + "completions/min_terminated_length": 32.0, + "epoch": 0.27276606639924894, + "grad_norm": 1.8710473775863647, + "kl": 6.328125, + "learning_rate": 9.20682096315646e-07, + "loss": 0.4437, + "num_tokens": 500256904.0, + "reward": 0.873046875, + "reward_std": 0.3145484924316406, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.81640625, + "rewards/tag_count_reward/std": 0.2564898729324341, + "step": 799 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 772.85546875, + "completions/mean_terminated_length": 729.0626831054688, + "completions/min_length": 38.0, + "completions/min_terminated_length": 38.0, + "epoch": 0.27310745071263975, + "grad_norm": 2.3094871044158936, + "kl": 6.1484375, + "learning_rate": 9.20377875849142e-07, + "loss": 0.4261, + "num_tokens": 500728510.0, + "reward": 0.85205078125, + "reward_std": 0.30268627405166626, + "rewards/accuracy_reward/mean": 0.038306452333927155, + "rewards/accuracy_reward/std": 0.19212883710861206, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.81298828125, + "rewards/tag_count_reward/std": 0.2627609074115753, + "step": 800 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2002.0, + "completions/mean_length": 739.9609375, + "completions/mean_terminated_length": 708.5680541992188, + "completions/min_length": 48.0, + "completions/min_terminated_length": 48.0, + "epoch": 0.27344883502603057, + "grad_norm": 7.303294658660889, + "kl": 5.828125, + "learning_rate": 9.200731296989862e-07, + "loss": 0.3461, + "num_tokens": 501184426.0, + "reward": 0.88720703125, + "reward_std": 0.32939833402633667, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.81298828125, + "rewards/tag_count_reward/std": 0.2599530518054962, + "step": 801 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1981.0, + "completions/mean_length": 784.236328125, + "completions/mean_terminated_length": 730.1853637695312, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "epoch": 0.2737902193394214, + "grad_norm": 2.827252149581909, + "kl": 5.921875, + "learning_rate": 9.1976785829771e-07, + "loss": 0.3723, + "num_tokens": 501662051.0, + "reward": 0.85205078125, + "reward_std": 0.2874138653278351, + "rewards/accuracy_reward/mean": 0.033203125, + "rewards/accuracy_reward/std": 0.17934183776378632, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.81884765625, + "rewards/tag_count_reward/std": 0.254165381193161, + "step": 802 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2030.0, + "completions/mean_length": 764.15625, + "completions/mean_terminated_length": 701.016357421875, + "completions/min_length": 41.0, + "completions/min_terminated_length": 41.0, + "epoch": 0.27413160365281214, + "grad_norm": 2.5225069522857666, + "kl": 6.3671875, + "learning_rate": 9.194620620785905e-07, + "loss": 0.3984, + "num_tokens": 502127811.0, + "reward": 0.8857421875, + "reward_std": 0.29661011695861816, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8271484375, + "rewards/tag_count_reward/std": 0.2505478858947754, + "step": 803 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2015.0, + "completions/mean_length": 765.015625, + "completions/mean_terminated_length": 693.591796875, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "epoch": 0.27447298796620295, + "grad_norm": 4.072855472564697, + "kl": 5.7421875, + "learning_rate": 9.191557414756495e-07, + "loss": 0.4049, + "num_tokens": 502591579.0, + "reward": 0.890625, + "reward_std": 0.33126142621040344, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.8203125, + "rewards/tag_count_reward/std": 0.25687703490257263, + "step": 804 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.056640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 828.080078125, + "completions/mean_terminated_length": 754.8344116210938, + "completions/min_length": 119.0, + "completions/min_terminated_length": 119.0, + "epoch": 0.27481437227959377, + "grad_norm": 3.144808292388916, + "kl": 6.0546875, + "learning_rate": 9.188488969236531e-07, + "loss": 0.4617, + "num_tokens": 503088020.0, + "reward": 0.9384765625, + "reward_std": 0.3468049168586731, + "rewards/accuracy_reward/mean": 0.109375, + "rewards/accuracy_reward/std": 0.31241437792778015, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.8271484375, + "rewards/tag_count_reward/std": 0.2558613717556, + "step": 805 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2033.0, + "completions/mean_length": 761.431640625, + "completions/mean_terminated_length": 703.6672973632812, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.2751557565929846, + "grad_norm": 6.194130897521973, + "kl": 5.5703125, + "learning_rate": 9.185415288581105e-07, + "loss": 0.4197, + "num_tokens": 503560961.0, + "reward": 0.875, + "reward_std": 0.2672116756439209, + "rewards/accuracy_reward/mean": 0.02734375, + "rewards/accuracy_reward/std": 0.16324250400066376, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.845703125, + "rewards/tag_count_reward/std": 0.23641379177570343, + "step": 806 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2029.0, + "completions/mean_length": 783.896484375, + "completions/mean_terminated_length": 721.7274169921875, + "completions/min_length": 16.0, + "completions/min_terminated_length": 16.0, + "epoch": 0.27549714090637534, + "grad_norm": 1.6844385862350464, + "kl": 5.296875, + "learning_rate": 9.182336377152753e-07, + "loss": 0.3712, + "num_tokens": 504045100.0, + "reward": 0.8623046875, + "reward_std": 0.2624799311161041, + "rewards/accuracy_reward/mean": 0.017578125, + "rewards/accuracy_reward/std": 0.13154059648513794, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8447265625, + "rewards/tag_count_reward/std": 0.24493204057216644, + "step": 807 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 769.341796875, + "completions/mean_terminated_length": 720.0628662109375, + "completions/min_length": 8.0, + "completions/min_terminated_length": 8.0, + "epoch": 0.27583852521976615, + "grad_norm": 6.979933261871338, + "kl": 7.3515625, + "learning_rate": 9.179252239321419e-07, + "loss": 0.4488, + "num_tokens": 504514219.0, + "reward": 0.9267578125, + "reward_std": 0.32439467310905457, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.8388671875, + "rewards/tag_count_reward/std": 0.24262726306915283, + "step": 808 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2013.0, + "completions/mean_length": 801.63671875, + "completions/mean_terminated_length": 756.2227172851562, + "completions/min_length": 141.0, + "completions/min_terminated_length": 141.0, + "epoch": 0.27617990953315696, + "grad_norm": 2.551192045211792, + "kl": 7.0390625, + "learning_rate": 9.176162879464477e-07, + "loss": 0.4806, + "num_tokens": 505010033.0, + "reward": 0.93994140625, + "reward_std": 0.34775638580322266, + "rewards/accuracy_reward/mean": 0.099609375, + "rewards/accuracy_reward/std": 0.29977133870124817, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.83642578125, + "rewards/tag_count_reward/std": 0.25551894307136536, + "step": 809 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 789.384765625, + "completions/mean_terminated_length": 732.87548828125, + "completions/min_length": 131.0, + "completions/min_terminated_length": 131.0, + "epoch": 0.2765212938465478, + "grad_norm": 8.129801750183105, + "kl": 8.7734375, + "learning_rate": 9.173068301966707e-07, + "loss": 0.564, + "num_tokens": 505487574.0, + "reward": 0.884765625, + "reward_std": 0.3085659146308899, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.830078125, + "rewards/tag_count_reward/std": 0.2530317008495331, + "step": 810 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1959.0, + "completions/mean_length": 745.357421875, + "completions/mean_terminated_length": 714.0940551757812, + "completions/min_length": 130.0, + "completions/min_terminated_length": 130.0, + "epoch": 0.27686267815993854, + "grad_norm": 2.361401319503784, + "kl": 5.078125, + "learning_rate": 9.169968511220296e-07, + "loss": 0.3309, + "num_tokens": 505946237.0, + "reward": 0.90771484375, + "reward_std": 0.2869418263435364, + "rewards/accuracy_reward/mean": 0.05443548411130905, + "rewards/accuracy_reward/std": 0.227104052901268, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.85498046875, + "rewards/tag_count_reward/std": 0.23111572861671448, + "step": 811 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1926.0, + "completions/mean_length": 702.703125, + "completions/mean_terminated_length": 667.6553344726562, + "completions/min_length": 165.0, + "completions/min_terminated_length": 165.0, + "epoch": 0.27720406247332935, + "grad_norm": 4.859031677246094, + "kl": 4.73828125, + "learning_rate": 9.166863511624828e-07, + "loss": 0.3554, + "num_tokens": 506385845.0, + "reward": 0.98681640625, + "reward_std": 0.3073740005493164, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.88134765625, + "rewards/tag_count_reward/std": 0.2103225141763687, + "step": 812 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2014.0, + "completions/mean_length": 759.708984375, + "completions/mean_terminated_length": 720.8269653320312, + "completions/min_length": 44.0, + "completions/min_terminated_length": 44.0, + "epoch": 0.27754544678672016, + "grad_norm": 4.14418363571167, + "kl": 3.7109375, + "learning_rate": 9.163753307587285e-07, + "loss": 0.2662, + "num_tokens": 506852368.0, + "reward": 0.90771484375, + "reward_std": 0.27509641647338867, + "rewards/accuracy_reward/mean": 0.029296875, + "rewards/accuracy_reward/std": 0.16880230605602264, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.87646484375, + "rewards/tag_count_reward/std": 0.23250487446784973, + "step": 813 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2011.0, + "completions/mean_length": 716.064453125, + "completions/mean_terminated_length": 692.2325439453125, + "completions/min_length": 83.0, + "completions/min_terminated_length": 83.0, + "epoch": 0.277886831100111, + "grad_norm": 8.611891746520996, + "kl": 4.66796875, + "learning_rate": 9.160637903522031e-07, + "loss": 0.3462, + "num_tokens": 507292577.0, + "reward": 0.90771484375, + "reward_std": 0.28996092081069946, + "rewards/accuracy_reward/mean": 0.04233871027827263, + "rewards/accuracy_reward/std": 0.2015640139579773, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.86669921875, + "rewards/tag_count_reward/std": 0.2291809767484665, + "step": 814 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1904.0, + "completions/mean_length": 752.1640625, + "completions/mean_terminated_length": 726.3506469726562, + "completions/min_length": 70.0, + "completions/min_terminated_length": 70.0, + "epoch": 0.27822821541350173, + "grad_norm": 2.8570358753204346, + "kl": 4.58984375, + "learning_rate": 9.157517303850814e-07, + "loss": 0.2891, + "num_tokens": 507759973.0, + "reward": 0.93505859375, + "reward_std": 0.27268046140670776, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.87255859375, + "rewards/tag_count_reward/std": 0.2222767174243927, + "step": 815 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2007.0, + "completions/mean_length": 757.0625, + "completions/mean_terminated_length": 712.727294921875, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "epoch": 0.27856959972689255, + "grad_norm": 6.3649797439575195, + "kl": 6.4140625, + "learning_rate": 9.154391513002754e-07, + "loss": 0.3876, + "num_tokens": 508231173.0, + "reward": 0.8984375, + "reward_std": 0.2914305627346039, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.84765625, + "rewards/tag_count_reward/std": 0.24377650022506714, + "step": 816 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1885.0, + "completions/mean_length": 680.474609375, + "completions/mean_terminated_length": 661.5188598632812, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "epoch": 0.27891098404028336, + "grad_norm": 2.3406496047973633, + "kl": 5.4921875, + "learning_rate": 9.151260535414336e-07, + "loss": 0.3474, + "num_tokens": 508655576.0, + "reward": 0.9384765625, + "reward_std": 0.2707592844963074, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.8759765625, + "rewards/tag_count_reward/std": 0.2127285748720169, + "step": 817 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.009765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2003.0, + "completions/mean_length": 653.947265625, + "completions/mean_terminated_length": 640.19921875, + "completions/min_length": 113.0, + "completions/min_terminated_length": 113.0, + "epoch": 0.2792523683536742, + "grad_norm": 10.013449668884277, + "kl": 5.6484375, + "learning_rate": 9.148124375529414e-07, + "loss": 0.2774, + "num_tokens": 509063469.0, + "reward": 0.9892578125, + "reward_std": 0.3207172453403473, + "rewards/accuracy_reward/mean": 0.1015625, + "rewards/accuracy_reward/std": 0.30236753821372986, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.8818359375, + "rewards/tag_count_reward/std": 0.21030718088150024, + "step": 818 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1917.0, + "completions/mean_length": 724.18359375, + "completions/mean_terminated_length": 708.4862060546875, + "completions/min_length": 138.0, + "completions/min_terminated_length": 138.0, + "epoch": 0.27959375266706493, + "grad_norm": 6.756846904754639, + "kl": 5.28125, + "learning_rate": 9.144983037799192e-07, + "loss": 0.2523, + "num_tokens": 509515339.0, + "reward": 0.9443359375, + "reward_std": 0.27607595920562744, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8798828125, + "rewards/tag_count_reward/std": 0.20919561386108398, + "step": 819 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1849.0, + "completions/mean_length": 681.810546875, + "completions/mean_terminated_length": 660.1250610351562, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "epoch": 0.27993513698045575, + "grad_norm": 2.1358535289764404, + "kl": 4.38671875, + "learning_rate": 9.141836526682226e-07, + "loss": 0.2262, + "num_tokens": 509936922.0, + "reward": 0.96435546875, + "reward_std": 0.3120565414428711, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.88427734375, + "rewards/tag_count_reward/std": 0.21195176243782043, + "step": 820 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1929.0, + "completions/mean_length": 726.5546875, + "completions/mean_terminated_length": 700.2310791015625, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "epoch": 0.28027652129384656, + "grad_norm": 1.8945742845535278, + "kl": 4.0, + "learning_rate": 9.138684846644408e-07, + "loss": 0.1925, + "num_tokens": 510383462.0, + "reward": 0.95458984375, + "reward_std": 0.28026509284973145, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.87646484375, + "rewards/tag_count_reward/std": 0.21783897280693054, + "step": 821 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1948.0, + "completions/mean_length": 728.2421875, + "completions/mean_terminated_length": 709.9485473632812, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.2806179056072374, + "grad_norm": 7.067627906799316, + "kl": 2.88671875, + "learning_rate": 9.135528002158977e-07, + "loss": 0.2135, + "num_tokens": 510837762.0, + "reward": 0.9560546875, + "reward_std": 0.2493462711572647, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8974609375, + "rewards/tag_count_reward/std": 0.19278420507907867, + "step": 822 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1948.0, + "completions/mean_length": 703.02734375, + "completions/mean_terminated_length": 681.6785888671875, + "completions/min_length": 122.0, + "completions/min_terminated_length": 122.0, + "epoch": 0.28095928992062813, + "grad_norm": 6.042422771453857, + "kl": 1.919921875, + "learning_rate": 9.132365997706493e-07, + "loss": 0.1195, + "num_tokens": 511277152.0, + "reward": 1.02490234375, + "reward_std": 0.23825043439865112, + "rewards/accuracy_reward/mean": 0.107421875, + "rewards/accuracy_reward/std": 0.30995169281959534, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.91748046875, + "rewards/tag_count_reward/std": 0.17458952963352203, + "step": 823 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2030.0, + "completions/mean_length": 681.3203125, + "completions/mean_terminated_length": 670.55908203125, + "completions/min_length": 141.0, + "completions/min_terminated_length": 141.0, + "epoch": 0.28130067423401894, + "grad_norm": 6.865614891052246, + "kl": 2.017578125, + "learning_rate": 9.129198837774846e-07, + "loss": 0.1366, + "num_tokens": 511708660.0, + "reward": 0.994140625, + "reward_std": 0.2854694128036499, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.908203125, + "rewards/tag_count_reward/std": 0.1943957805633545, + "step": 824 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1960.0, + "completions/mean_length": 752.12109375, + "completions/mean_terminated_length": 731.5516357421875, + "completions/min_length": 18.0, + "completions/min_terminated_length": 18.0, + "epoch": 0.28164205854740976, + "grad_norm": 3.810732364654541, + "kl": 2.841796875, + "learning_rate": 9.126026526859236e-07, + "loss": 0.1974, + "num_tokens": 512170514.0, + "reward": 0.97314453125, + "reward_std": 0.2556039094924927, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.90673828125, + "rewards/tag_count_reward/std": 0.19274641573429108, + "step": 825 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.001953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1886.0, + "completions/mean_length": 694.3203125, + "completions/mean_terminated_length": 691.6712036132812, + "completions/min_length": 157.0, + "completions/min_terminated_length": 157.0, + "epoch": 0.28198344286080057, + "grad_norm": 2.4061853885650635, + "kl": 3.23046875, + "learning_rate": 9.122849069462181e-07, + "loss": 0.1637, + "num_tokens": 512600934.0, + "reward": 0.97998046875, + "reward_std": 0.2752445340156555, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.90576171875, + "rewards/tag_count_reward/std": 0.19542469084262848, + "step": 826 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1625.0, + "completions/mean_length": 655.1875, + "completions/mean_terminated_length": 649.7255249023438, + "completions/min_length": 18.0, + "completions/min_terminated_length": 18.0, + "epoch": 0.28232482717419133, + "grad_norm": 2.6872496604919434, + "kl": 2.453125, + "learning_rate": 9.119666470093501e-07, + "loss": 0.0745, + "num_tokens": 513029270.0, + "reward": 1.03125, + "reward_std": 0.26225048303604126, + "rewards/accuracy_reward/mean": 0.11328125, + "rewards/accuracy_reward/std": 0.3172462284564972, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.9140625, + "rewards/tag_count_reward/std": 0.18489299714565277, + "step": 827 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2009.0, + "completions/mean_length": 718.95703125, + "completions/mean_terminated_length": 713.7451171875, + "completions/min_length": 20.0, + "completions/min_terminated_length": 20.0, + "epoch": 0.28266621148758214, + "grad_norm": 2.1618266105651855, + "kl": 3.9921875, + "learning_rate": 9.116478733270312e-07, + "loss": 0.1946, + "num_tokens": 513477536.0, + "reward": 0.9921875, + "reward_std": 0.2947632074356079, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.896484375, + "rewards/tag_count_reward/std": 0.19974872469902039, + "step": 828 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.005859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2025.0, + "completions/mean_length": 739.02734375, + "completions/mean_terminated_length": 731.3123779296875, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "epoch": 0.28300759580097296, + "grad_norm": 5.863043308258057, + "kl": 4.46875, + "learning_rate": 9.113285863517024e-07, + "loss": 0.1952, + "num_tokens": 513935646.0, + "reward": 0.9384765625, + "reward_std": 0.26553913950920105, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.8955078125, + "rewards/tag_count_reward/std": 0.2010718286037445, + "step": 829 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.001953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1931.0, + "completions/mean_length": 774.67578125, + "completions/mean_terminated_length": 772.1839599609375, + "completions/min_length": 230.0, + "completions/min_terminated_length": 230.0, + "epoch": 0.28334898011436377, + "grad_norm": 1.5997233390808105, + "kl": 3.85546875, + "learning_rate": 9.110087865365333e-07, + "loss": 0.2139, + "num_tokens": 514414040.0, + "reward": 0.96435546875, + "reward_std": 0.23553423583507538, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.91162109375, + "rewards/tag_count_reward/std": 0.1867077797651291, + "step": 830 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2005.0, + "completions/mean_length": 815.685546875, + "completions/mean_terminated_length": 781.0421142578125, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "epoch": 0.2836903644277545, + "grad_norm": 1.3619939088821411, + "kl": 4.3125, + "learning_rate": 9.10688474335421e-07, + "loss": 0.2463, + "num_tokens": 514906375.0, + "reward": 0.93408203125, + "reward_std": 0.2167353332042694, + "rewards/accuracy_reward/mean": 0.02734375, + "rewards/accuracy_reward/std": 0.16324250400066376, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.90673828125, + "rewards/tag_count_reward/std": 0.1952681541442871, + "step": 831 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.009765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1968.0, + "completions/mean_length": 716.037109375, + "completions/mean_terminated_length": 702.9013671875, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "epoch": 0.28403174874114534, + "grad_norm": 3.041804552078247, + "kl": 3.1796875, + "learning_rate": 9.103676502029901e-07, + "loss": 0.216, + "num_tokens": 515353962.0, + "reward": 1.046875, + "reward_std": 0.2764724791049957, + "rewards/accuracy_reward/mean": 0.13750000298023224, + "rewards/accuracy_reward/std": 0.34473371505737305, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.91796875, + "rewards/tag_count_reward/std": 0.1813446581363678, + "step": 832 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1927.0, + "completions/mean_length": 773.44140625, + "completions/mean_terminated_length": 755.7742919921875, + "completions/min_length": 183.0, + "completions/min_terminated_length": 183.0, + "epoch": 0.28437313305453615, + "grad_norm": 1.7975035905838013, + "kl": 3.40625, + "learning_rate": 9.100463145945921e-07, + "loss": 0.1994, + "num_tokens": 515842252.0, + "reward": 0.9501953125, + "reward_std": 0.22530975937843323, + "rewards/accuracy_reward/mean": 0.0463709682226181, + "rewards/accuracy_reward/std": 0.21049949526786804, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.9033203125, + "rewards/tag_count_reward/std": 0.1951684206724167, + "step": 833 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.005859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 715.888671875, + "completions/mean_terminated_length": 708.037353515625, + "completions/min_length": 129.0, + "completions/min_terminated_length": 129.0, + "epoch": 0.28471451736792697, + "grad_norm": 3.068521738052368, + "kl": 3.193359375, + "learning_rate": 9.097244679663037e-07, + "loss": 0.2008, + "num_tokens": 516292371.0, + "reward": 1.033203125, + "reward_std": 0.2703208327293396, + "rewards/accuracy_reward/mean": 0.115234375, + "rewards/accuracy_reward/std": 0.3196168541908264, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.91796875, + "rewards/tag_count_reward/std": 0.18268859386444092, + "step": 834 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.009765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2024.0, + "completions/mean_length": 718.63671875, + "completions/mean_terminated_length": 705.526611328125, + "completions/min_length": 107.0, + "completions/min_terminated_length": 107.0, + "epoch": 0.2850559016813177, + "grad_norm": 2.360788345336914, + "kl": 4.5390625, + "learning_rate": 9.094021107749277e-07, + "loss": 0.2485, + "num_tokens": 516736937.0, + "reward": 1.04931640625, + "reward_std": 0.3307260274887085, + "rewards/accuracy_reward/mean": 0.146484375, + "rewards/accuracy_reward/std": 0.35393697023391724, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.90087890625, + "rewards/tag_count_reward/std": 0.20646168291568756, + "step": 835 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.009765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1888.0, + "completions/mean_length": 684.75390625, + "completions/mean_terminated_length": 671.3096923828125, + "completions/min_length": 61.0, + "completions/min_terminated_length": 61.0, + "epoch": 0.28539728599470854, + "grad_norm": 3.68979549407959, + "kl": 4.734375, + "learning_rate": 9.090792434779911e-07, + "loss": 0.2605, + "num_tokens": 517166475.0, + "reward": 1.048828125, + "reward_std": 0.32015591859817505, + "rewards/accuracy_reward/mean": 0.138671875, + "rewards/accuracy_reward/std": 0.34594178199768066, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.90625, + "rewards/tag_count_reward/std": 0.19345958530902863, + "step": 836 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1984.0, + "completions/mean_length": 767.984375, + "completions/mean_terminated_length": 745.0814819335938, + "completions/min_length": 71.0, + "completions/min_terminated_length": 71.0, + "epoch": 0.28573867030809935, + "grad_norm": 3.829807996749878, + "kl": 5.890625, + "learning_rate": 9.087558665337447e-07, + "loss": 0.3743, + "num_tokens": 517634995.0, + "reward": 0.99169921875, + "reward_std": 0.3004174530506134, + "rewards/accuracy_reward/mean": 0.1015625, + "rewards/accuracy_reward/std": 0.30236753821372986, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.88623046875, + "rewards/tag_count_reward/std": 0.21811513602733612, + "step": 837 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 752.056640625, + "completions/mean_terminated_length": 728.8687744140625, + "completions/min_length": 142.0, + "completions/min_terminated_length": 142.0, + "epoch": 0.28608005462149017, + "grad_norm": 3.1375844478607178, + "kl": 3.765625, + "learning_rate": 9.084319804011631e-07, + "loss": 0.2343, + "num_tokens": 518098624.0, + "reward": 0.9423828125, + "reward_std": 0.2559373080730438, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9013671875, + "rewards/tag_count_reward/std": 0.20401567220687866, + "step": 838 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1996.0, + "completions/mean_length": 757.591796875, + "completions/mean_terminated_length": 726.6220092773438, + "completions/min_length": 160.0, + "completions/min_terminated_length": 160.0, + "epoch": 0.2864214389348809, + "grad_norm": 2.0383784770965576, + "kl": 4.453125, + "learning_rate": 9.081075855399434e-07, + "loss": 0.2651, + "num_tokens": 518557599.0, + "reward": 1.0068359375, + "reward_std": 0.31170833110809326, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.8974609375, + "rewards/tag_count_reward/std": 0.2115394026041031, + "step": 839 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1992.0, + "completions/mean_length": 701.892578125, + "completions/mean_terminated_length": 685.9308471679688, + "completions/min_length": 133.0, + "completions/min_terminated_length": 133.0, + "epoch": 0.28676282324827174, + "grad_norm": 3.5112719535827637, + "kl": 3.80078125, + "learning_rate": 9.077826824105049e-07, + "loss": 0.2563, + "num_tokens": 518990744.0, + "reward": 0.96142578125, + "reward_std": 0.2386517971754074, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.91650390625, + "rewards/tag_count_reward/std": 0.18033477663993835, + "step": 840 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1908.0, + "completions/mean_length": 749.23828125, + "completions/mean_terminated_length": 715.40283203125, + "completions/min_length": 205.0, + "completions/min_terminated_length": 205.0, + "epoch": 0.28710420756166255, + "grad_norm": 2.8488667011260986, + "kl": 4.8046875, + "learning_rate": 9.074572714739881e-07, + "loss": 0.3292, + "num_tokens": 519449026.0, + "reward": 0.94873046875, + "reward_std": 0.24731914699077606, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.90185546875, + "rewards/tag_count_reward/std": 0.19474875926971436, + "step": 841 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2016.0, + "completions/mean_length": 689.671875, + "completions/mean_terminated_length": 657.072021484375, + "completions/min_length": 108.0, + "completions/min_terminated_length": 108.0, + "epoch": 0.28744559187505336, + "grad_norm": 2.858069896697998, + "kl": 5.765625, + "learning_rate": 9.071313531922541e-07, + "loss": 0.3456, + "num_tokens": 519869770.0, + "reward": 1.013671875, + "reward_std": 0.328676700592041, + "rewards/accuracy_reward/mean": 0.12890625, + "rewards/accuracy_reward/std": 0.33542385697364807, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.8828125, + "rewards/tag_count_reward/std": 0.21714095771312714, + "step": 842 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2003.0, + "completions/mean_length": 757.505859375, + "completions/mean_terminated_length": 729.171630859375, + "completions/min_length": 175.0, + "completions/min_terminated_length": 175.0, + "epoch": 0.2877869761884441, + "grad_norm": 1.8919070959091187, + "kl": 5.265625, + "learning_rate": 9.068049280278846e-07, + "loss": 0.3371, + "num_tokens": 520338189.0, + "reward": 0.9970703125, + "reward_std": 0.27113696932792664, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.9013671875, + "rewards/tag_count_reward/std": 0.20639976859092712, + "step": 843 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2011.0, + "completions/mean_length": 808.9765625, + "completions/mean_terminated_length": 763.8299560546875, + "completions/min_length": 210.0, + "completions/min_terminated_length": 210.0, + "epoch": 0.28812836050183493, + "grad_norm": 3.263883590698242, + "kl": 6.1015625, + "learning_rate": 9.064779964441802e-07, + "loss": 0.3948, + "num_tokens": 520829713.0, + "reward": 0.99609375, + "reward_std": 0.31634271144866943, + "rewards/accuracy_reward/mean": 0.107421875, + "rewards/accuracy_reward/std": 0.30995169281959534, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.884765625, + "rewards/tag_count_reward/std": 0.22426731884479523, + "step": 844 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1881.0, + "completions/mean_length": 673.951171875, + "completions/mean_terminated_length": 657.6581420898438, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "epoch": 0.28846974481522575, + "grad_norm": 2.096222400665283, + "kl": 3.875, + "learning_rate": 9.061505589051606e-07, + "loss": 0.2581, + "num_tokens": 521249912.0, + "reward": 0.99169921875, + "reward_std": 0.24171248078346252, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.92529296875, + "rewards/tag_count_reward/std": 0.17035679519176483, + "step": 845 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 737.296875, + "completions/mean_terminated_length": 716.4921264648438, + "completions/min_length": 184.0, + "completions/min_terminated_length": 184.0, + "epoch": 0.28881112912861656, + "grad_norm": 4.202604293823242, + "kl": 4.58203125, + "learning_rate": 9.058226158755634e-07, + "loss": 0.3379, + "num_tokens": 521700464.0, + "reward": 0.990234375, + "reward_std": 0.2905173897743225, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.904296875, + "rewards/tag_count_reward/std": 0.20240987837314606, + "step": 846 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 783.7734375, + "completions/mean_terminated_length": 756.0159912109375, + "completions/min_length": 63.0, + "completions/min_terminated_length": 63.0, + "epoch": 0.2891525134420073, + "grad_norm": 1.9926021099090576, + "kl": 5.2109375, + "learning_rate": 9.05494167820844e-07, + "loss": 0.364, + "num_tokens": 522177132.0, + "reward": 0.98291015625, + "reward_std": 0.28996962308883667, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.89501953125, + "rewards/tag_count_reward/std": 0.21004518866539001, + "step": 847 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2004.0, + "completions/mean_length": 751.43359375, + "completions/mean_terminated_length": 730.8532104492188, + "completions/min_length": 173.0, + "completions/min_terminated_length": 173.0, + "epoch": 0.28949389775539813, + "grad_norm": 2.8588552474975586, + "kl": 4.7890625, + "learning_rate": 9.05165215207174e-07, + "loss": 0.2968, + "num_tokens": 522645802.0, + "reward": 0.966796875, + "reward_std": 0.2664114832878113, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.90625, + "rewards/tag_count_reward/std": 0.19155354797840118, + "step": 848 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1965.0, + "completions/mean_length": 723.662109375, + "completions/mean_terminated_length": 702.6409301757812, + "completions/min_length": 174.0, + "completions/min_terminated_length": 174.0, + "epoch": 0.28983528206878895, + "grad_norm": 2.4607505798339844, + "kl": 5.546875, + "learning_rate": 9.048357585014417e-07, + "loss": 0.3355, + "num_tokens": 523092253.0, + "reward": 1.01220703125, + "reward_std": 0.2797544598579407, + "rewards/accuracy_reward/mean": 0.099609375, + "rewards/accuracy_reward/std": 0.29977133870124817, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.91064453125, + "rewards/tag_count_reward/std": 0.19333051145076752, + "step": 849 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1995.0, + "completions/mean_length": 716.376953125, + "completions/mean_terminated_length": 689.8506469726562, + "completions/min_length": 43.0, + "completions/min_terminated_length": 43.0, + "epoch": 0.29017666638217976, + "grad_norm": 9.725028038024902, + "kl": 5.46875, + "learning_rate": 9.045057981712504e-07, + "loss": 0.293, + "num_tokens": 523538302.0, + "reward": 1.00537109375, + "reward_std": 0.3139130473136902, + "rewards/accuracy_reward/mean": 0.0927419364452362, + "rewards/accuracy_reward/std": 0.2903633117675781, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.90966796875, + "rewards/tag_count_reward/std": 0.20216365158557892, + "step": 850 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 727.6640625, + "completions/mean_terminated_length": 701.362548828125, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "epoch": 0.2905180506955705, + "grad_norm": 2.414006233215332, + "kl": 5.2578125, + "learning_rate": 9.041753346849187e-07, + "loss": 0.3348, + "num_tokens": 523989938.0, + "reward": 0.94482421875, + "reward_std": 0.2373579442501068, + "rewards/accuracy_reward/mean": 0.03515625, + "rewards/accuracy_reward/std": 0.1843547374010086, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.90576171875, + "rewards/tag_count_reward/std": 0.19729334115982056, + "step": 851 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1994.0, + "completions/mean_length": 717.421875, + "completions/mean_terminated_length": 682.7575073242188, + "completions/min_length": 202.0, + "completions/min_terminated_length": 202.0, + "epoch": 0.29085943500896133, + "grad_norm": 5.3849053382873535, + "kl": 4.13671875, + "learning_rate": 9.038443685114791e-07, + "loss": 0.3105, + "num_tokens": 524439306.0, + "reward": 1.01953125, + "reward_std": 0.2760043144226074, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.923828125, + "rewards/tag_count_reward/std": 0.18053138256072998, + "step": 852 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1967.0, + "completions/mean_length": 720.787109375, + "completions/mean_terminated_length": 699.7202758789062, + "completions/min_length": 84.0, + "completions/min_terminated_length": 84.0, + "epoch": 0.29120081932235214, + "grad_norm": 4.139332294464111, + "kl": 3.68359375, + "learning_rate": 9.035129001206771e-07, + "loss": 0.2529, + "num_tokens": 524883133.0, + "reward": 1.017578125, + "reward_std": 0.2673831582069397, + "rewards/accuracy_reward/mean": 0.10483870655298233, + "rewards/accuracy_reward/std": 0.30665475130081177, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.9140625, + "rewards/tag_count_reward/std": 0.18882036209106445, + "step": 853 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1906.0, + "completions/mean_length": 711.49609375, + "completions/mean_terminated_length": 682.1516723632812, + "completions/min_length": 10.0, + "completions/min_terminated_length": 10.0, + "epoch": 0.29154220363574296, + "grad_norm": 3.6715269088745117, + "kl": 3.7109375, + "learning_rate": 9.03180929982972e-07, + "loss": 0.2463, + "num_tokens": 525314395.0, + "reward": 0.9794921875, + "reward_std": 0.2377520203590393, + "rewards/accuracy_reward/mean": 0.06451612710952759, + "rewards/accuracy_reward/std": 0.2459181249141693, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9169921875, + "rewards/tag_count_reward/std": 0.1849110871553421, + "step": 854 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.009765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1942.0, + "completions/mean_length": 730.3515625, + "completions/mean_terminated_length": 717.3569946289062, + "completions/min_length": 141.0, + "completions/min_terminated_length": 141.0, + "epoch": 0.2918835879491337, + "grad_norm": 1.6833791732788086, + "kl": 4.6171875, + "learning_rate": 9.028484585695345e-07, + "loss": 0.3114, + "num_tokens": 525762831.0, + "reward": 0.9453125, + "reward_std": 0.23490703105926514, + "rewards/accuracy_reward/mean": 0.03515625, + "rewards/accuracy_reward/std": 0.1843547374010086, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.91015625, + "rewards/tag_count_reward/std": 0.19151362776756287, + "step": 855 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2024.0, + "completions/mean_length": 735.22265625, + "completions/mean_terminated_length": 724.8858032226562, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "epoch": 0.29222497226252453, + "grad_norm": 4.737995624542236, + "kl": 3.63671875, + "learning_rate": 9.025154863522467e-07, + "loss": 0.1956, + "num_tokens": 526217937.0, + "reward": 1.00146484375, + "reward_std": 0.2507244944572449, + "rewards/accuracy_reward/mean": 0.0947580635547638, + "rewards/accuracy_reward/std": 0.29317617416381836, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.90966796875, + "rewards/tag_count_reward/std": 0.1890321522951126, + "step": 856 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2021.0, + "completions/mean_length": 770.6171875, + "completions/mean_terminated_length": 724.0728759765625, + "completions/min_length": 169.0, + "completions/min_terminated_length": 169.0, + "epoch": 0.29256635657591534, + "grad_norm": 5.21553373336792, + "kl": 5.40625, + "learning_rate": 9.021820138037022e-07, + "loss": 0.3029, + "num_tokens": 526697133.0, + "reward": 1.0234375, + "reward_std": 0.3139427900314331, + "rewards/accuracy_reward/mean": 0.130859375, + "rewards/accuracy_reward/std": 0.33757632970809937, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.892578125, + "rewards/tag_count_reward/std": 0.19828914105892181, + "step": 857 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2031.0, + "completions/mean_length": 715.330078125, + "completions/mean_terminated_length": 694.1766357421875, + "completions/min_length": 152.0, + "completions/min_terminated_length": 152.0, + "epoch": 0.29290774088930616, + "grad_norm": 1.9480489492416382, + "kl": 4.60546875, + "learning_rate": 9.01848041397204e-07, + "loss": 0.2503, + "num_tokens": 527136118.0, + "reward": 0.962890625, + "reward_std": 0.24685801565647125, + "rewards/accuracy_reward/mean": 0.058467742055654526, + "rewards/accuracy_reward/std": 0.23486268520355225, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.90625, + "rewards/tag_count_reward/std": 0.2009030431509018, + "step": 858 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1904.0, + "completions/mean_length": 706.712890625, + "completions/mean_terminated_length": 690.808349609375, + "completions/min_length": 8.0, + "completions/min_terminated_length": 8.0, + "epoch": 0.2932491252026969, + "grad_norm": 1.5688979625701904, + "kl": 3.5625, + "learning_rate": 9.015135696067649e-07, + "loss": 0.1984, + "num_tokens": 527576643.0, + "reward": 0.97412109375, + "reward_std": 0.21969881653785706, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.92529296875, + "rewards/tag_count_reward/std": 0.18282388150691986, + "step": 859 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.009765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2010.0, + "completions/mean_length": 730.560546875, + "completions/mean_terminated_length": 717.5680541992188, + "completions/min_length": 170.0, + "completions/min_terminated_length": 170.0, + "epoch": 0.2935905095160877, + "grad_norm": 1.75925874710083, + "kl": 2.9765625, + "learning_rate": 9.011785989071066e-07, + "loss": 0.1618, + "num_tokens": 528023202.0, + "reward": 1.01318359375, + "reward_std": 0.25561508536338806, + "rewards/accuracy_reward/mean": 0.09072580933570862, + "rewards/accuracy_reward/std": 0.2875087857246399, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.92333984375, + "rewards/tag_count_reward/std": 0.17305581271648407, + "step": 860 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 729.85546875, + "completions/mean_terminated_length": 708.9325561523438, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "epoch": 0.29393189382947854, + "grad_norm": 3.2166786193847656, + "kl": 3.55859375, + "learning_rate": 9.008431297736585e-07, + "loss": 0.2405, + "num_tokens": 528467288.0, + "reward": 1.02685546875, + "reward_std": 0.28205960988998413, + "rewards/accuracy_reward/mean": 0.1015625, + "rewards/accuracy_reward/std": 0.30236753821372986, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.92333984375, + "rewards/tag_count_reward/std": 0.1840168684720993, + "step": 861 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1915.0, + "completions/mean_length": 709.005859375, + "completions/mean_terminated_length": 698.4625854492188, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "epoch": 0.29427327814286935, + "grad_norm": 3.0324487686157227, + "kl": 2.662109375, + "learning_rate": 9.005071626825577e-07, + "loss": 0.1677, + "num_tokens": 528906331.0, + "reward": 1.02099609375, + "reward_std": 0.25581175088882446, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.92724609375, + "rewards/tag_count_reward/std": 0.16831976175308228, + "step": 862 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1963.0, + "completions/mean_length": 723.283203125, + "completions/mean_terminated_length": 707.5751342773438, + "completions/min_length": 59.0, + "completions/min_terminated_length": 59.0, + "epoch": 0.2946146624562601, + "grad_norm": 2.2073960304260254, + "kl": 4.109375, + "learning_rate": 9.001706981106482e-07, + "loss": 0.2688, + "num_tokens": 529349116.0, + "reward": 1.0986328125, + "reward_std": 0.3106450140476227, + "rewards/accuracy_reward/mean": 0.166015625, + "rewards/accuracy_reward/std": 0.3724585771560669, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.9287109375, + "rewards/tag_count_reward/std": 0.17074689269065857, + "step": 863 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.005859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1884.0, + "completions/mean_length": 731.982421875, + "completions/mean_terminated_length": 724.2259521484375, + "completions/min_length": 110.0, + "completions/min_terminated_length": 110.0, + "epoch": 0.2949560467696509, + "grad_norm": 1.7147998809814453, + "kl": 4.4140625, + "learning_rate": 8.998337365354798e-07, + "loss": 0.2635, + "num_tokens": 529803779.0, + "reward": 1.00927734375, + "reward_std": 0.27351608872413635, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.009765625, + "rewards/format_reward/std": 0.09843364357948303, + "rewards/tag_count_reward/mean": 0.92138671875, + "rewards/tag_count_reward/std": 0.17359058558940887, + "step": 864 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1800.0, + "completions/mean_length": 750.14453125, + "completions/mean_terminated_length": 729.543701171875, + "completions/min_length": 161.0, + "completions/min_terminated_length": 161.0, + "epoch": 0.29529743108304174, + "grad_norm": 4.473459243774414, + "kl": 4.53125, + "learning_rate": 8.994962784353079e-07, + "loss": 0.2475, + "num_tokens": 530265709.0, + "reward": 0.9765625, + "reward_std": 0.24417805671691895, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.90625, + "rewards/tag_count_reward/std": 0.19282633066177368, + "step": 865 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.009765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 720.380859375, + "completions/mean_terminated_length": 707.2879638671875, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "epoch": 0.29563881539643255, + "grad_norm": 3.271315336227417, + "kl": 4.3828125, + "learning_rate": 8.991583242890924e-07, + "loss": 0.2248, + "num_tokens": 530709632.0, + "reward": 1.00537109375, + "reward_std": 0.25619742274284363, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.91748046875, + "rewards/tag_count_reward/std": 0.17805784940719604, + "step": 866 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1905.0, + "completions/mean_length": 748.685546875, + "completions/mean_terminated_length": 733.2786865234375, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "epoch": 0.2959801997098233, + "grad_norm": 3.136951208114624, + "kl": 4.26953125, + "learning_rate": 8.988198745764976e-07, + "loss": 0.2058, + "num_tokens": 531177903.0, + "reward": 0.96044921875, + "reward_std": 0.26861572265625, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.90185546875, + "rewards/tag_count_reward/std": 0.20692861080169678, + "step": 867 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2017.0, + "completions/mean_length": 749.0078125, + "completions/mean_terminated_length": 723.1314697265625, + "completions/min_length": 134.0, + "completions/min_terminated_length": 134.0, + "epoch": 0.2963215840232141, + "grad_norm": 3.2323830127716064, + "kl": 2.6015625, + "learning_rate": 8.984809297778908e-07, + "loss": 0.1476, + "num_tokens": 531646547.0, + "reward": 0.95751953125, + "reward_std": 0.2431613653898239, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.90869140625, + "rewards/tag_count_reward/std": 0.19177725911140442, + "step": 868 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.009765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1984.0, + "completions/mean_length": 759.9921875, + "completions/mean_terminated_length": 747.2899780273438, + "completions/min_length": 173.0, + "completions/min_terminated_length": 173.0, + "epoch": 0.29666296833660494, + "grad_norm": 1.4316259622573853, + "kl": 2.7890625, + "learning_rate": 8.981414903743423e-07, + "loss": 0.1238, + "num_tokens": 532112527.0, + "reward": 0.9697265625, + "reward_std": 0.24940599501132965, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.9091796875, + "rewards/tag_count_reward/std": 0.1865164041519165, + "step": 869 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1997.0, + "completions/mean_length": 747.65625, + "completions/mean_terminated_length": 729.6317138671875, + "completions/min_length": 123.0, + "completions/min_terminated_length": 123.0, + "epoch": 0.29700435264999575, + "grad_norm": 2.9305245876312256, + "kl": 2.56640625, + "learning_rate": 8.97801556847624e-07, + "loss": 0.1271, + "num_tokens": 532564767.0, + "reward": 1.0126953125, + "reward_std": 0.2796088457107544, + "rewards/accuracy_reward/mean": 0.1015625, + "rewards/accuracy_reward/std": 0.30236753821372986, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.9072265625, + "rewards/tag_count_reward/std": 0.19705891609191895, + "step": 870 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1794.0, + "completions/mean_length": 787.693359375, + "completions/mean_terminated_length": 762.587646484375, + "completions/min_length": 27.0, + "completions/min_terminated_length": 27.0, + "epoch": 0.2973457369633865, + "grad_norm": 4.79686164855957, + "kl": 2.396484375, + "learning_rate": 8.974611296802096e-07, + "loss": 0.173, + "num_tokens": 533046402.0, + "reward": 0.982421875, + "reward_std": 0.2162947952747345, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.927734375, + "rewards/tag_count_reward/std": 0.1696154922246933, + "step": 871 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.009765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1917.0, + "completions/mean_length": 740.326171875, + "completions/mean_terminated_length": 727.4299926757812, + "completions/min_length": 166.0, + "completions/min_terminated_length": 166.0, + "epoch": 0.2976871212767773, + "grad_norm": 2.1338069438934326, + "kl": 2.603515625, + "learning_rate": 8.971202093552731e-07, + "loss": 0.169, + "num_tokens": 533502761.0, + "reward": 0.99951171875, + "reward_std": 0.271058589220047, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.91552734375, + "rewards/tag_count_reward/std": 0.1771378070116043, + "step": 872 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.005859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1949.0, + "completions/mean_length": 706.083984375, + "completions/mean_terminated_length": 698.1748657226562, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "epoch": 0.29802850559016814, + "grad_norm": 4.597038745880127, + "kl": 2.5234375, + "learning_rate": 8.967787963566887e-07, + "loss": 0.1367, + "num_tokens": 533941108.0, + "reward": 1.0380859375, + "reward_std": 0.2726272642612457, + "rewards/accuracy_reward/mean": 0.123046875, + "rewards/accuracy_reward/std": 0.32881227135658264, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9150390625, + "rewards/tag_count_reward/std": 0.18731388449668884, + "step": 873 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1844.0, + "completions/mean_length": 755.275390625, + "completions/mean_terminated_length": 739.9466552734375, + "completions/min_length": 16.0, + "completions/min_terminated_length": 16.0, + "epoch": 0.29836988990355895, + "grad_norm": 0.946505069732666, + "kl": 2.72265625, + "learning_rate": 8.964368911690296e-07, + "loss": 0.1177, + "num_tokens": 534400353.0, + "reward": 0.974609375, + "reward_std": 0.24221986532211304, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.9140625, + "rewards/tag_count_reward/std": 0.1822277307510376, + "step": 874 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1928.0, + "completions/mean_length": 749.759765625, + "completions/mean_terminated_length": 731.764404296875, + "completions/min_length": 134.0, + "completions/min_terminated_length": 134.0, + "epoch": 0.2987112742169497, + "grad_norm": 4.6282057762146, + "kl": 3.8125, + "learning_rate": 8.960944942775675e-07, + "loss": 0.1705, + "num_tokens": 534876454.0, + "reward": 1.01416015625, + "reward_std": 0.26385408639907837, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.92431640625, + "rewards/tag_count_reward/std": 0.1713578850030899, + "step": 875 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1994.0, + "completions/mean_length": 717.359375, + "completions/mean_terminated_length": 706.8818969726562, + "completions/min_length": 118.0, + "completions/min_terminated_length": 118.0, + "epoch": 0.2990526585303405, + "grad_norm": 5.376757621765137, + "kl": 4.5859375, + "learning_rate": 8.957516061682724e-07, + "loss": 0.2, + "num_tokens": 535316814.0, + "reward": 0.9931640625, + "reward_std": 0.2733707129955292, + "rewards/accuracy_reward/mean": 0.1015625, + "rewards/accuracy_reward/std": 0.30236753821372986, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.8916015625, + "rewards/tag_count_reward/std": 0.20741577446460724, + "step": 876 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1849.0, + "completions/mean_length": 678.541015625, + "completions/mean_terminated_length": 659.5584106445312, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "epoch": 0.29939404284373133, + "grad_norm": 2.8910319805145264, + "kl": 4.19140625, + "learning_rate": 8.954082273278112e-07, + "loss": 0.229, + "num_tokens": 535744499.0, + "reward": 0.98779296875, + "reward_std": 0.26597756147384644, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.90576171875, + "rewards/tag_count_reward/std": 0.20036904513835907, + "step": 877 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.005859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1665.0, + "completions/mean_length": 761.509765625, + "completions/mean_terminated_length": 753.9273071289062, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "epoch": 0.29973542715712215, + "grad_norm": 3.734360694885254, + "kl": 3.84765625, + "learning_rate": 8.950643582435474e-07, + "loss": 0.1729, + "num_tokens": 536209304.0, + "reward": 0.98095703125, + "reward_std": 0.27370643615722656, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.017578125, + "rewards/format_reward/std": 0.13154059648513794, + "rewards/tag_count_reward/mean": 0.91455078125, + "rewards/tag_count_reward/std": 0.18806926906108856, + "step": 878 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 709.39453125, + "completions/mean_terminated_length": 698.8543090820312, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "epoch": 0.3000768114705129, + "grad_norm": 1.302786111831665, + "kl": 2.28515625, + "learning_rate": 8.9471999940354e-07, + "loss": 0.1255, + "num_tokens": 536660162.0, + "reward": 1.046875, + "reward_std": 0.2732976973056793, + "rewards/accuracy_reward/mean": 0.1088709682226181, + "rewards/accuracy_reward/std": 0.31179171800613403, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.939453125, + "rewards/tag_count_reward/std": 0.15794700384140015, + "step": 879 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.009765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1938.0, + "completions/mean_length": 777.19921875, + "completions/mean_terminated_length": 764.6666870117188, + "completions/min_length": 195.0, + "completions/min_terminated_length": 195.0, + "epoch": 0.3004181957839037, + "grad_norm": 3.1007723808288574, + "kl": 2.30859375, + "learning_rate": 8.943751512965437e-07, + "loss": 0.1278, + "num_tokens": 537154328.0, + "reward": 1.0029296875, + "reward_std": 0.23788556456565857, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9365234375, + "rewards/tag_count_reward/std": 0.16440613567829132, + "step": 880 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1622.0, + "completions/mean_length": 667.107421875, + "completions/mean_terminated_length": 656.2342529296875, + "completions/min_length": 25.0, + "completions/min_terminated_length": 25.0, + "epoch": 0.30075958009729453, + "grad_norm": 3.776506185531616, + "kl": 1.515625, + "learning_rate": 8.940298144120074e-07, + "loss": 0.104, + "num_tokens": 537567663.0, + "reward": 1.07275390625, + "reward_std": 0.2482367753982544, + "rewards/accuracy_reward/mean": 0.109375, + "rewards/accuracy_reward/std": 0.31241437792778015, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.95751953125, + "rewards/tag_count_reward/std": 0.13733947277069092, + "step": 881 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1971.0, + "completions/mean_length": 761.78515625, + "completions/mean_terminated_length": 738.7713623046875, + "completions/min_length": 174.0, + "completions/min_terminated_length": 174.0, + "epoch": 0.30110096441068535, + "grad_norm": 1.399491310119629, + "kl": 2.953125, + "learning_rate": 8.936839892400732e-07, + "loss": 0.1791, + "num_tokens": 538037409.0, + "reward": 1.07373046875, + "reward_std": 0.3079867362976074, + "rewards/accuracy_reward/mean": 0.12890625, + "rewards/accuracy_reward/std": 0.33542385697364807, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.93896484375, + "rewards/tag_count_reward/std": 0.16716907918453217, + "step": 882 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1799.0, + "completions/mean_length": 818.599609375, + "completions/mean_terminated_length": 804.0217895507812, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "epoch": 0.3014423487240761, + "grad_norm": 3.5203235149383545, + "kl": 2.138671875, + "learning_rate": 8.93337676271577e-07, + "loss": 0.1362, + "num_tokens": 538535348.0, + "reward": 1.0615234375, + "reward_std": 0.24921491742134094, + "rewards/accuracy_reward/mean": 0.111328125, + "rewards/accuracy_reward/std": 0.31484565138816833, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.9482421875, + "rewards/tag_count_reward/std": 0.14757278561592102, + "step": 883 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.009765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1845.0, + "completions/mean_length": 756.958984375, + "completions/mean_terminated_length": 744.226806640625, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "epoch": 0.3017837330374669, + "grad_norm": 1.0830971002578735, + "kl": 2.2578125, + "learning_rate": 8.929908759980467e-07, + "loss": 0.1088, + "num_tokens": 539001279.0, + "reward": 1.01953125, + "reward_std": 0.25354111194610596, + "rewards/accuracy_reward/mean": 0.08266129344701767, + "rewards/accuracy_reward/std": 0.2756475806236267, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.939453125, + "rewards/tag_count_reward/std": 0.16025325655937195, + "step": 884 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 740.443359375, + "completions/mean_terminated_length": 724.9387817382812, + "completions/min_length": 7.0, + "completions/min_terminated_length": 7.0, + "epoch": 0.30212511735085773, + "grad_norm": 2.659788131713867, + "kl": 3.5234375, + "learning_rate": 8.926435889117019e-07, + "loss": 0.1967, + "num_tokens": 539454610.0, + "reward": 1.0634765625, + "reward_std": 0.2688376307487488, + "rewards/accuracy_reward/mean": 0.125, + "rewards/accuracy_reward/std": 0.3310528099536896, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9423828125, + "rewards/tag_count_reward/std": 0.16208821535110474, + "step": 885 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.001953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2013.0, + "completions/mean_length": 695.8828125, + "completions/mean_terminated_length": 693.2367553710938, + "completions/min_length": 83.0, + "completions/min_terminated_length": 83.0, + "epoch": 0.30246650166424854, + "grad_norm": 3.676887035369873, + "kl": 4.10546875, + "learning_rate": 8.922958155054527e-07, + "loss": 0.2062, + "num_tokens": 539886326.0, + "reward": 1.02001953125, + "reward_std": 0.26067638397216797, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.93408203125, + "rewards/tag_count_reward/std": 0.17183120548725128, + "step": 886 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 699.330078125, + "completions/mean_terminated_length": 680.6356811523438, + "completions/min_length": 25.0, + "completions/min_terminated_length": 25.0, + "epoch": 0.3028078859776393, + "grad_norm": 6.330347537994385, + "kl": 4.8984375, + "learning_rate": 8.919475562729004e-07, + "loss": 0.2743, + "num_tokens": 540331279.0, + "reward": 1.0302734375, + "reward_std": 0.28771382570266724, + "rewards/accuracy_reward/mean": 0.111328125, + "rewards/accuracy_reward/std": 0.31484565138816833, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.9169921875, + "rewards/tag_count_reward/std": 0.18753820657730103, + "step": 887 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2015.0, + "completions/mean_length": 756.12109375, + "completions/mean_terminated_length": 717.1307373046875, + "completions/min_length": 112.0, + "completions/min_terminated_length": 112.0, + "epoch": 0.3031492702910301, + "grad_norm": 5.038356304168701, + "kl": 3.91796875, + "learning_rate": 8.915988117083351e-07, + "loss": 0.2365, + "num_tokens": 540804093.0, + "reward": 0.9609375, + "reward_std": 0.21736985445022583, + "rewards/accuracy_reward/mean": 0.037109375, + "rewards/accuracy_reward/std": 0.18921469151973724, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.921875, + "rewards/tag_count_reward/std": 0.17832663655281067, + "step": 888 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2017.0, + "completions/mean_length": 698.138671875, + "completions/mean_terminated_length": 668.5009765625, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "epoch": 0.30349065460442093, + "grad_norm": 2.2378509044647217, + "kl": 4.10546875, + "learning_rate": 8.912495823067356e-07, + "loss": 0.2691, + "num_tokens": 541235924.0, + "reward": 1.0390625, + "reward_std": 0.252128005027771, + "rewards/accuracy_reward/mean": 0.10685484111309052, + "rewards/accuracy_reward/std": 0.30924052000045776, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.935546875, + "rewards/tag_count_reward/std": 0.16843964159488678, + "step": 889 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1981.0, + "completions/mean_length": 722.943359375, + "completions/mean_terminated_length": 701.9107666015625, + "completions/min_length": 9.0, + "completions/min_terminated_length": 9.0, + "epoch": 0.30383203891781174, + "grad_norm": 4.78679895401001, + "kl": 3.390625, + "learning_rate": 8.908998685637696e-07, + "loss": 0.2461, + "num_tokens": 541677319.0, + "reward": 1.05322265625, + "reward_std": 0.2654269337654114, + "rewards/accuracy_reward/mean": 0.111328125, + "rewards/accuracy_reward/std": 0.31484565138816833, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.93994140625, + "rewards/tag_count_reward/std": 0.16531828045845032, + "step": 890 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1810.0, + "completions/mean_length": 801.087890625, + "completions/mean_terminated_length": 742.4396362304688, + "completions/min_length": 242.0, + "completions/min_terminated_length": 242.0, + "epoch": 0.3041734232312025, + "grad_norm": 7.234511375427246, + "kl": 4.8125, + "learning_rate": 8.905496709757917e-07, + "loss": 0.3774, + "num_tokens": 542164548.0, + "reward": 0.9912109375, + "reward_std": 0.27330106496810913, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9169921875, + "rewards/tag_count_reward/std": 0.19395042955875397, + "step": 891 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1940.0, + "completions/mean_length": 770.48046875, + "completions/mean_terminated_length": 734.5662231445312, + "completions/min_length": 150.0, + "completions/min_terminated_length": 150.0, + "epoch": 0.3045148075445933, + "grad_norm": 3.368119955062866, + "kl": 4.32421875, + "learning_rate": 8.90198990039843e-07, + "loss": 0.2899, + "num_tokens": 542626170.0, + "reward": 0.96630859375, + "reward_std": 0.23860511183738708, + "rewards/accuracy_reward/mean": 0.052419353276491165, + "rewards/accuracy_reward/std": 0.22309619188308716, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.91162109375, + "rewards/tag_count_reward/std": 0.19377975165843964, + "step": 892 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1949.0, + "completions/mean_length": 731.869140625, + "completions/mean_terminated_length": 700.2820434570312, + "completions/min_length": 29.0, + "completions/min_terminated_length": 29.0, + "epoch": 0.3048561918579841, + "grad_norm": 3.3479795455932617, + "kl": 3.55859375, + "learning_rate": 8.898478262536513e-07, + "loss": 0.1946, + "num_tokens": 543078279.0, + "reward": 1.009765625, + "reward_std": 0.2593262195587158, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.927734375, + "rewards/tag_count_reward/std": 0.18677493929862976, + "step": 893 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2019.0, + "completions/mean_length": 782.36328125, + "completions/mean_terminated_length": 757.1514282226562, + "completions/min_length": 62.0, + "completions/min_terminated_length": 62.0, + "epoch": 0.30519757617137494, + "grad_norm": 2.381742477416992, + "kl": 3.73046875, + "learning_rate": 8.894961801156292e-07, + "loss": 0.1785, + "num_tokens": 543560577.0, + "reward": 1.02294921875, + "reward_std": 0.29584503173828125, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.92724609375, + "rewards/tag_count_reward/std": 0.18024997413158417, + "step": 894 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1905.0, + "completions/mean_length": 769.951171875, + "completions/mean_terminated_length": 739.2780151367188, + "completions/min_length": 15.0, + "completions/min_terminated_length": 15.0, + "epoch": 0.3055389604847657, + "grad_norm": 2.093144655227661, + "kl": 4.19921875, + "learning_rate": 8.891440521248742e-07, + "loss": 0.2567, + "num_tokens": 544023448.0, + "reward": 1.08447265625, + "reward_std": 0.2756364345550537, + "rewards/accuracy_reward/mean": 0.146484375, + "rewards/accuracy_reward/std": 0.35393697023391724, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.93408203125, + "rewards/tag_count_reward/std": 0.1704016625881195, + "step": 895 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1998.0, + "completions/mean_length": 769.62109375, + "completions/mean_terminated_length": 738.9400634765625, + "completions/min_length": 156.0, + "completions/min_terminated_length": 156.0, + "epoch": 0.3058803447981565, + "grad_norm": 1.946558952331543, + "kl": 4.0078125, + "learning_rate": 8.887914427811676e-07, + "loss": 0.2431, + "num_tokens": 544490358.0, + "reward": 1.00439453125, + "reward_std": 0.24462217092514038, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.0078125, + "rewards/format_reward/std": 0.08812850713729858, + "rewards/tag_count_reward/mean": 0.93212890625, + "rewards/tag_count_reward/std": 0.16818346083164215, + "step": 896 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.009765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2000.0, + "completions/mean_length": 781.759765625, + "completions/mean_terminated_length": 769.272216796875, + "completions/min_length": 163.0, + "completions/min_terminated_length": 163.0, + "epoch": 0.3062217291115473, + "grad_norm": 1.506081461906433, + "kl": 2.919921875, + "learning_rate": 8.884383525849736e-07, + "loss": 0.1543, + "num_tokens": 544980043.0, + "reward": 1.029296875, + "reward_std": 0.2404770702123642, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.9296875, + "rewards/tag_count_reward/std": 0.17043600976467133, + "step": 897 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1922.0, + "completions/mean_length": 779.76171875, + "completions/mean_terminated_length": 757.0695190429688, + "completions/min_length": 248.0, + "completions/min_terminated_length": 248.0, + "epoch": 0.30656311342493814, + "grad_norm": 1.7713485956192017, + "kl": 3.1015625, + "learning_rate": 8.880847820374395e-07, + "loss": 0.1911, + "num_tokens": 545451569.0, + "reward": 1.01708984375, + "reward_std": 0.22191838920116425, + "rewards/accuracy_reward/mean": 0.07500000298023224, + "rewards/accuracy_reward/std": 0.26366615295410156, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.94482421875, + "rewards/tag_count_reward/std": 0.15004098415374756, + "step": 898 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1772.0, + "completions/mean_length": 726.482421875, + "completions/mean_terminated_length": 716.0767822265625, + "completions/min_length": 158.0, + "completions/min_terminated_length": 158.0, + "epoch": 0.3069044977383289, + "grad_norm": 1.510651707649231, + "kl": 2.796875, + "learning_rate": 8.877307316403936e-07, + "loss": 0.1611, + "num_tokens": 545899336.0, + "reward": 0.99267578125, + "reward_std": 0.26471325755119324, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.93212890625, + "rewards/tag_count_reward/std": 0.17319931089878082, + "step": 899 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1952.0, + "completions/mean_length": 716.80078125, + "completions/mean_terminated_length": 711.5804443359375, + "completions/min_length": 18.0, + "completions/min_terminated_length": 18.0, + "epoch": 0.3072458820517197, + "grad_norm": 1.4322909116744995, + "kl": 3.0078125, + "learning_rate": 8.87376201896346e-07, + "loss": 0.205, + "num_tokens": 546336914.0, + "reward": 0.9833984375, + "reward_std": 0.19191157817840576, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.9423828125, + "rewards/tag_count_reward/std": 0.15593473613262177, + "step": 900 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2013.0, + "completions/mean_length": 818.115234375, + "completions/mean_terminated_length": 793.6155395507812, + "completions/min_length": 195.0, + "completions/min_terminated_length": 195.0, + "epoch": 0.3075872663651105, + "grad_norm": 1.4152556657791138, + "kl": 4.125, + "learning_rate": 8.870211933084868e-07, + "loss": 0.2409, + "num_tokens": 546841805.0, + "reward": 0.947265625, + "reward_std": 0.20224609971046448, + "rewards/accuracy_reward/mean": 0.017578125, + "rewards/accuracy_reward/std": 0.13154059648513794, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.92578125, + "rewards/tag_count_reward/std": 0.1806689351797104, + "step": 901 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 699.349609375, + "completions/mean_terminated_length": 683.3577270507812, + "completions/min_length": 14.0, + "completions/min_terminated_length": 14.0, + "epoch": 0.30792865067850134, + "grad_norm": 2.7752535343170166, + "kl": 3.45703125, + "learning_rate": 8.866657063806859e-07, + "loss": 0.2088, + "num_tokens": 547282480.0, + "reward": 1.00244140625, + "reward_std": 0.20316222310066223, + "rewards/accuracy_reward/mean": 0.06451612710952759, + "rewards/accuracy_reward/std": 0.2459181249141693, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.93994140625, + "rewards/tag_count_reward/std": 0.15774647891521454, + "step": 902 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1846.0, + "completions/mean_length": 693.265625, + "completions/mean_terminated_length": 677.2015991210938, + "completions/min_length": 82.0, + "completions/min_terminated_length": 82.0, + "epoch": 0.30827003499189215, + "grad_norm": 2.296919822692871, + "kl": 3.279296875, + "learning_rate": 8.863097416174916e-07, + "loss": 0.2034, + "num_tokens": 547717336.0, + "reward": 1.04296875, + "reward_std": 0.23253028094768524, + "rewards/accuracy_reward/mean": 0.1015625, + "rewards/accuracy_reward/std": 0.30236753821372986, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.94140625, + "rewards/tag_count_reward/std": 0.16021747887134552, + "step": 903 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 749.271484375, + "completions/mean_terminated_length": 739.0452880859375, + "completions/min_length": 143.0, + "completions/min_terminated_length": 143.0, + "epoch": 0.3086114193052829, + "grad_norm": 1.4624488353729248, + "kl": 3.84375, + "learning_rate": 8.859532995241309e-07, + "loss": 0.1991, + "num_tokens": 548182931.0, + "reward": 1.00341796875, + "reward_std": 0.2580622732639313, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.93115234375, + "rewards/tag_count_reward/std": 0.17351900041103363, + "step": 904 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1840.0, + "completions/mean_length": 712.423828125, + "completions/mean_terminated_length": 707.1863403320312, + "completions/min_length": 133.0, + "completions/min_terminated_length": 133.0, + "epoch": 0.3089528036186737, + "grad_norm": 2.0424282550811768, + "kl": 3.1484375, + "learning_rate": 8.855963806065085e-07, + "loss": 0.1681, + "num_tokens": 548622588.0, + "reward": 1.044921875, + "reward_std": 0.23043325543403625, + "rewards/accuracy_reward/mean": 0.107421875, + "rewards/accuracy_reward/std": 0.30995169281959534, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9375, + "rewards/tag_count_reward/std": 0.16102655231952667, + "step": 905 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1875.0, + "completions/mean_length": 734.583984375, + "completions/mean_terminated_length": 697.66064453125, + "completions/min_length": 130.0, + "completions/min_terminated_length": 130.0, + "epoch": 0.30929418793206453, + "grad_norm": 1.9697262048721313, + "kl": 4.66796875, + "learning_rate": 8.85238985371205e-07, + "loss": 0.295, + "num_tokens": 549079399.0, + "reward": 0.978515625, + "reward_std": 0.23790127038955688, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.919921875, + "rewards/tag_count_reward/std": 0.18685677647590637, + "step": 906 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1904.0, + "completions/mean_length": 788.373046875, + "completions/mean_terminated_length": 758.1420288085938, + "completions/min_length": 138.0, + "completions/min_terminated_length": 138.0, + "epoch": 0.30963557224545535, + "grad_norm": 2.7737860679626465, + "kl": 5.25, + "learning_rate": 8.848811143254779e-07, + "loss": 0.3418, + "num_tokens": 549560838.0, + "reward": 0.9716796875, + "reward_std": 0.23982642590999603, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.9169921875, + "rewards/tag_count_reward/std": 0.19645671546459198, + "step": 907 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 702.029296875, + "completions/mean_terminated_length": 677.9462890625, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "epoch": 0.3099769565588461, + "grad_norm": 5.901828289031982, + "kl": 4.921875, + "learning_rate": 8.845227679772596e-07, + "loss": 0.303, + "num_tokens": 549997317.0, + "reward": 1.03466796875, + "reward_std": 0.3133937120437622, + "rewards/accuracy_reward/mean": 0.1171875, + "rewards/accuracy_reward/std": 0.32195815443992615, + "rewards/format_reward/mean": 0.0078125, + "rewards/format_reward/std": 0.08812850713729858, + "rewards/tag_count_reward/mean": 0.90966796875, + "rewards/tag_count_reward/std": 0.20397058129310608, + "step": 908 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 690.173828125, + "completions/mean_terminated_length": 674.0731201171875, + "completions/min_length": 119.0, + "completions/min_terminated_length": 119.0, + "epoch": 0.3103183408722369, + "grad_norm": 2.511085271835327, + "kl": 3.02734375, + "learning_rate": 8.841639468351571e-07, + "loss": 0.1876, + "num_tokens": 550424270.0, + "reward": 1.0078125, + "reward_std": 0.252117395401001, + "rewards/accuracy_reward/mean": 0.06653226166963577, + "rewards/accuracy_reward/std": 0.2494617998600006, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.9375, + "rewards/tag_count_reward/std": 0.1540389060974121, + "step": 909 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2025.0, + "completions/mean_length": 744.2109375, + "completions/mean_terminated_length": 718.2390747070312, + "completions/min_length": 128.0, + "completions/min_terminated_length": 128.0, + "epoch": 0.31065972518562773, + "grad_norm": 3.7137417793273926, + "kl": 3.53125, + "learning_rate": 8.838046514084516e-07, + "loss": 0.2435, + "num_tokens": 550879002.0, + "reward": 1.0166015625, + "reward_std": 0.2949827313423157, + "rewards/accuracy_reward/mean": 0.099609375, + "rewards/accuracy_reward/std": 0.29977133870124817, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.9111328125, + "rewards/tag_count_reward/std": 0.20069128274917603, + "step": 910 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 751.1796875, + "completions/mean_terminated_length": 725.3466186523438, + "completions/min_length": 78.0, + "completions/min_terminated_length": 78.0, + "epoch": 0.31100110949901855, + "grad_norm": 7.024240970611572, + "kl": 2.826171875, + "learning_rate": 8.834448822070971e-07, + "loss": 0.2561, + "num_tokens": 551344150.0, + "reward": 0.96875, + "reward_std": 0.2577192187309265, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.921875, + "rewards/tag_count_reward/std": 0.1921909898519516, + "step": 911 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2031.0, + "completions/mean_length": 740.40625, + "completions/mean_terminated_length": 690.0121459960938, + "completions/min_length": 110.0, + "completions/min_terminated_length": 110.0, + "epoch": 0.3113424938124093, + "grad_norm": 7.939428329467773, + "kl": 3.85546875, + "learning_rate": 8.830846397417202e-07, + "loss": 0.3628, + "num_tokens": 551798710.0, + "reward": 1.0, + "reward_std": 0.24333296716213226, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "rewards/tag_count_reward/mean": 0.91796875, + "rewards/tag_count_reward/std": 0.19499453902244568, + "step": 912 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2003.0, + "completions/mean_length": 709.05859375, + "completions/mean_terminated_length": 690.4990234375, + "completions/min_length": 194.0, + "completions/min_terminated_length": 194.0, + "epoch": 0.3116838781258001, + "grad_norm": 2.98781681060791, + "kl": 3.34765625, + "learning_rate": 8.827239245236194e-07, + "loss": 0.2729, + "num_tokens": 552234644.0, + "reward": 1.041015625, + "reward_std": 0.21898271143436432, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.9453125, + "rewards/tag_count_reward/std": 0.15143637359142303, + "step": 913 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1994.0, + "completions/mean_length": 731.732421875, + "completions/mean_terminated_length": 710.83935546875, + "completions/min_length": 71.0, + "completions/min_terminated_length": 71.0, + "epoch": 0.31202526243919093, + "grad_norm": 1.9520854949951172, + "kl": 4.39453125, + "learning_rate": 8.823627370647634e-07, + "loss": 0.2713, + "num_tokens": 552692043.0, + "reward": 0.9521484375, + "reward_std": 0.20709767937660217, + "rewards/accuracy_reward/mean": 0.01953125, + "rewards/accuracy_reward/std": 0.1385180652141571, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.9287109375, + "rewards/tag_count_reward/std": 0.18117332458496094, + "step": 914 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2004.0, + "completions/mean_length": 748.013671875, + "completions/mean_terminated_length": 724.75341796875, + "completions/min_length": 181.0, + "completions/min_terminated_length": 181.0, + "epoch": 0.31236664675258174, + "grad_norm": 4.948241233825684, + "kl": 4.57421875, + "learning_rate": 8.820010778777925e-07, + "loss": 0.2511, + "num_tokens": 553156594.0, + "reward": 0.98681640625, + "reward_std": 0.23312708735466003, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.92822265625, + "rewards/tag_count_reward/std": 0.1792825609445572, + "step": 915 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1781.0, + "completions/mean_length": 742.59375, + "completions/mean_terminated_length": 719.236572265625, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "epoch": 0.3127080310659725, + "grad_norm": 5.643583297729492, + "kl": 5.015625, + "learning_rate": 8.816389474760151e-07, + "loss": 0.2985, + "num_tokens": 553628130.0, + "reward": 1.0224609375, + "reward_std": 0.2850502133369446, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.9228515625, + "rewards/tag_count_reward/std": 0.19194971024990082, + "step": 916 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1971.0, + "completions/mean_length": 721.310546875, + "completions/mean_terminated_length": 705.5791015625, + "completions/min_length": 140.0, + "completions/min_terminated_length": 140.0, + "epoch": 0.3130494153793633, + "grad_norm": 3.775449514389038, + "kl": 4.61328125, + "learning_rate": 8.812763463734095e-07, + "loss": 0.2438, + "num_tokens": 554074641.0, + "reward": 0.990234375, + "reward_std": 0.244065523147583, + "rewards/accuracy_reward/mean": 0.06451612710952759, + "rewards/accuracy_reward/std": 0.2459181249141693, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.923828125, + "rewards/tag_count_reward/std": 0.18652920424938202, + "step": 917 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.005859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 746.1796875, + "completions/mean_terminated_length": 738.5068969726562, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "epoch": 0.31339079969275413, + "grad_norm": 3.482968330383301, + "kl": 4.94140625, + "learning_rate": 8.809132750846214e-07, + "loss": 0.2725, + "num_tokens": 554530477.0, + "reward": 1.064453125, + "reward_std": 0.3038240075111389, + "rewards/accuracy_reward/mean": 0.12890625, + "rewards/accuracy_reward/std": 0.33542385697364807, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.931640625, + "rewards/tag_count_reward/std": 0.1726529598236084, + "step": 918 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.009765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1877.0, + "completions/mean_length": 713.826171875, + "completions/mean_terminated_length": 700.6686401367188, + "completions/min_length": 76.0, + "completions/min_terminated_length": 76.0, + "epoch": 0.31373218400614494, + "grad_norm": 1.567186713218689, + "kl": 4.05078125, + "learning_rate": 8.805497341249642e-07, + "loss": 0.2236, + "num_tokens": 554972932.0, + "reward": 1.0068359375, + "reward_std": 0.24477024376392365, + "rewards/accuracy_reward/mean": 0.06854838877916336, + "rewards/accuracy_reward/std": 0.25293970108032227, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.9345703125, + "rewards/tag_count_reward/std": 0.17378656566143036, + "step": 919 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1838.0, + "completions/mean_length": 830.228515625, + "completions/mean_terminated_length": 795.9939575195312, + "completions/min_length": 162.0, + "completions/min_terminated_length": 162.0, + "epoch": 0.3140735683195357, + "grad_norm": 1.707174301147461, + "kl": 4.45703125, + "learning_rate": 8.801857240104179e-07, + "loss": 0.2765, + "num_tokens": 555473417.0, + "reward": 0.9892578125, + "reward_std": 0.26355940103530884, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.24230584502220154, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.9228515625, + "rewards/tag_count_reward/std": 0.18612663447856903, + "step": 920 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.009765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1885.0, + "completions/mean_length": 718.58203125, + "completions/mean_terminated_length": 705.471435546875, + "completions/min_length": 101.0, + "completions/min_terminated_length": 101.0, + "epoch": 0.3144149526329265, + "grad_norm": 1.327146053314209, + "kl": 3.484375, + "learning_rate": 8.798212452576282e-07, + "loss": 0.2167, + "num_tokens": 555922771.0, + "reward": 1.099609375, + "reward_std": 0.27388161420822144, + "rewards/accuracy_reward/mean": 0.15234375, + "rewards/accuracy_reward/std": 0.35970520973205566, + "rewards/format_reward/mean": 0.0078125, + "rewards/format_reward/std": 0.08812850713729858, + "rewards/tag_count_reward/mean": 0.939453125, + "rewards/tag_count_reward/std": 0.16327762603759766, + "step": 921 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2029.0, + "completions/mean_length": 830.103515625, + "completions/mean_terminated_length": 800.8740234375, + "completions/min_length": 127.0, + "completions/min_terminated_length": 127.0, + "epoch": 0.3147563369463173, + "grad_norm": 2.0233733654022217, + "kl": 3.1015625, + "learning_rate": 8.794562983839058e-07, + "loss": 0.2271, + "num_tokens": 556432488.0, + "reward": 1.06103515625, + "reward_std": 0.2801864445209503, + "rewards/accuracy_reward/mean": 0.1171875, + "rewards/accuracy_reward/std": 0.32195815443992615, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.93798828125, + "rewards/tag_count_reward/std": 0.16384941339492798, + "step": 922 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2001.0, + "completions/mean_length": 749.255859375, + "completions/mean_terminated_length": 723.3844604492188, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "epoch": 0.31509772125970814, + "grad_norm": 5.790834903717041, + "kl": 2.75390625, + "learning_rate": 8.790908839072262e-07, + "loss": 0.2092, + "num_tokens": 556894331.0, + "reward": 0.984375, + "reward_std": 0.20769117772579193, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.93359375, + "rewards/tag_count_reward/std": 0.16913031041622162, + "step": 923 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2005.0, + "completions/mean_length": 790.607421875, + "completions/mean_terminated_length": 752.6578979492188, + "completions/min_length": 18.0, + "completions/min_terminated_length": 18.0, + "epoch": 0.3154391055730989, + "grad_norm": 4.157425880432129, + "kl": 3.03515625, + "learning_rate": 8.787250023462286e-07, + "loss": 0.2037, + "num_tokens": 557371538.0, + "reward": 0.95654296875, + "reward_std": 0.19840088486671448, + "rewards/accuracy_reward/mean": 0.025390625, + "rewards/accuracy_reward/std": 0.15746226906776428, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.92529296875, + "rewards/tag_count_reward/std": 0.17876482009887695, + "step": 924 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2025.0, + "completions/mean_length": 772.291015625, + "completions/mean_terminated_length": 749.4651489257812, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "epoch": 0.3157804898864897, + "grad_norm": 1.3573524951934814, + "kl": 2.732421875, + "learning_rate": 8.783586542202148e-07, + "loss": 0.1264, + "num_tokens": 557839879.0, + "reward": 0.953125, + "reward_std": 0.24137933552265167, + "rewards/accuracy_reward/mean": 0.03427419438958168, + "rewards/accuracy_reward/std": 0.18211629986763, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.916015625, + "rewards/tag_count_reward/std": 0.19289569556713104, + "step": 925 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1916.0, + "completions/mean_length": 765.982421875, + "completions/mean_terminated_length": 743.043701171875, + "completions/min_length": 150.0, + "completions/min_terminated_length": 150.0, + "epoch": 0.3161218741998805, + "grad_norm": 1.8128553628921509, + "kl": 3.16015625, + "learning_rate": 8.779918400491488e-07, + "loss": 0.1972, + "num_tokens": 558313566.0, + "reward": 1.04443359375, + "reward_std": 0.29815906286239624, + "rewards/accuracy_reward/mean": 0.12109375, + "rewards/accuracy_reward/std": 0.3265552520751953, + "rewards/format_reward/mean": 0.0078125, + "rewards/format_reward/std": 0.08812850713729858, + "rewards/tag_count_reward/mean": 0.91552734375, + "rewards/tag_count_reward/std": 0.1865541785955429, + "step": 926 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1987.0, + "completions/mean_length": 746.015625, + "completions/mean_terminated_length": 722.7196655273438, + "completions/min_length": 237.0, + "completions/min_terminated_length": 237.0, + "epoch": 0.31646325851327134, + "grad_norm": 2.1845901012420654, + "kl": 3.37890625, + "learning_rate": 8.776245603536565e-07, + "loss": 0.2325, + "num_tokens": 558781126.0, + "reward": 0.99365234375, + "reward_std": 0.2560039162635803, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.0078125, + "rewards/format_reward/std": 0.08812850713729858, + "rewards/tag_count_reward/mean": 0.92529296875, + "rewards/tag_count_reward/std": 0.17876482009887695, + "step": 927 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1978.0, + "completions/mean_length": 767.728515625, + "completions/mean_terminated_length": 734.374755859375, + "completions/min_length": 160.0, + "completions/min_terminated_length": 160.0, + "epoch": 0.3168046428266621, + "grad_norm": 3.265126943588257, + "kl": 3.8828125, + "learning_rate": 8.772568156550241e-07, + "loss": 0.2827, + "num_tokens": 559256299.0, + "reward": 1.0185546875, + "reward_std": 0.2517598271369934, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.9345703125, + "rewards/tag_count_reward/std": 0.16878816485404968, + "step": 928 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0, + "completions/max_length": 1940.0, + "completions/max_terminated_length": 1940.0, + "completions/mean_length": 716.318359375, + "completions/mean_terminated_length": 716.318359375, + "completions/min_length": 113.0, + "completions/min_terminated_length": 113.0, + "epoch": 0.3171460271400529, + "grad_norm": 1.339988112449646, + "kl": 3.326171875, + "learning_rate": 8.76888606475198e-07, + "loss": 0.1828, + "num_tokens": 559695934.0, + "reward": 0.9970703125, + "reward_std": 0.2440335899591446, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.0078125, + "rewards/format_reward/std": 0.08812850713729858, + "rewards/tag_count_reward/mean": 0.9365234375, + "rewards/tag_count_reward/std": 0.16588735580444336, + "step": 929 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1992.0, + "completions/mean_length": 748.62109375, + "completions/mean_terminated_length": 717.43603515625, + "completions/min_length": 170.0, + "completions/min_terminated_length": 170.0, + "epoch": 0.3174874114534437, + "grad_norm": 6.6354146003723145, + "kl": 6.7890625, + "learning_rate": 8.765199333367837e-07, + "loss": 0.3981, + "num_tokens": 560155068.0, + "reward": 0.97216796875, + "reward_std": 0.25777316093444824, + "rewards/accuracy_reward/mean": 0.052419353276491165, + "rewards/accuracy_reward/std": 0.22309619188308716, + "rewards/format_reward/mean": 0.0078125, + "rewards/format_reward/std": 0.08812850713729858, + "rewards/tag_count_reward/mean": 0.91357421875, + "rewards/tag_count_reward/std": 0.19715769588947296, + "step": 930 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1979.0, + "completions/mean_length": 694.666015625, + "completions/mean_terminated_length": 659.4088134765625, + "completions/min_length": 62.0, + "completions/min_terminated_length": 62.0, + "epoch": 0.31782879576683454, + "grad_norm": 8.99532699584961, + "kl": 8.1484375, + "learning_rate": 8.761507967630453e-07, + "loss": 0.4642, + "num_tokens": 560595809.0, + "reward": 1.0048828125, + "reward_std": 0.27473604679107666, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.9091796875, + "rewards/tag_count_reward/std": 0.19796842336654663, + "step": 931 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 732.279296875, + "completions/mean_terminated_length": 706.0697631835938, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "epoch": 0.3181701800802253, + "grad_norm": 8.550848007202148, + "kl": 7.4921875, + "learning_rate": 8.757811972779048e-07, + "loss": 0.4005, + "num_tokens": 561045760.0, + "reward": 0.998046875, + "reward_std": 0.2719319462776184, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.0078125, + "rewards/format_reward/std": 0.08812850713729858, + "rewards/tag_count_reward/mean": 0.916015625, + "rewards/tag_count_reward/std": 0.18775463104248047, + "step": 932 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1938.0, + "completions/mean_length": 676.158203125, + "completions/mean_terminated_length": 659.891357421875, + "completions/min_length": 107.0, + "completions/min_terminated_length": 107.0, + "epoch": 0.3185115643936161, + "grad_norm": 8.880213737487793, + "kl": 6.3828125, + "learning_rate": 8.754111354059409e-07, + "loss": 0.3314, + "num_tokens": 561467201.0, + "reward": 0.99755859375, + "reward_std": 0.267604798078537, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.0078125, + "rewards/format_reward/std": 0.08812850713729858, + "rewards/tag_count_reward/mean": 0.92333984375, + "rewards/tag_count_reward/std": 0.17655426263809204, + "step": 933 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1937.0, + "completions/mean_length": 824.904296875, + "completions/mean_terminated_length": 790.5200805664062, + "completions/min_length": 130.0, + "completions/min_terminated_length": 130.0, + "epoch": 0.3188529487070069, + "grad_norm": 7.950473785400391, + "kl": 6.78125, + "learning_rate": 8.750406116723889e-07, + "loss": 0.3817, + "num_tokens": 561966576.0, + "reward": 0.9892578125, + "reward_std": 0.2762683928012848, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.009765625, + "rewards/format_reward/std": 0.09843364357948303, + "rewards/tag_count_reward/mean": 0.9033203125, + "rewards/tag_count_reward/std": 0.2007293701171875, + "step": 934 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 734.10546875, + "completions/mean_terminated_length": 705.2575073242188, + "completions/min_length": 154.0, + "completions/min_terminated_length": 154.0, + "epoch": 0.31919433302039774, + "grad_norm": 1.5691170692443848, + "kl": 4.28125, + "learning_rate": 8.746696266031392e-07, + "loss": 0.3072, + "num_tokens": 562428502.0, + "reward": 1.076171875, + "reward_std": 0.26259198784828186, + "rewards/accuracy_reward/mean": 0.12890625, + "rewards/accuracy_reward/std": 0.33542385697364807, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.943359375, + "rewards/tag_count_reward/std": 0.15785017609596252, + "step": 935 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2017.0, + "completions/mean_length": 815.935546875, + "completions/mean_terminated_length": 783.8377075195312, + "completions/min_length": 101.0, + "completions/min_terminated_length": 101.0, + "epoch": 0.3195357173337885, + "grad_norm": 4.007493495941162, + "kl": 4.50390625, + "learning_rate": 8.742981807247374e-07, + "loss": 0.3032, + "num_tokens": 562928613.0, + "reward": 0.9599609375, + "reward_std": 0.27960240840911865, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.9052734375, + "rewards/tag_count_reward/std": 0.214018315076828, + "step": 936 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1814.0, + "completions/mean_length": 679.859375, + "completions/mean_terminated_length": 658.1428833007812, + "completions/min_length": 146.0, + "completions/min_terminated_length": 146.0, + "epoch": 0.3198771016471793, + "grad_norm": 5.114379405975342, + "kl": 2.4375, + "learning_rate": 8.739262745643832e-07, + "loss": 0.1968, + "num_tokens": 563355053.0, + "reward": 1.02978515625, + "reward_std": 0.23758836090564728, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.94580078125, + "rewards/tag_count_reward/std": 0.1503971964120865, + "step": 937 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1886.0, + "completions/mean_length": 752.609375, + "completions/mean_terminated_length": 713.5130615234375, + "completions/min_length": 207.0, + "completions/min_terminated_length": 207.0, + "epoch": 0.3202184859605701, + "grad_norm": 8.205772399902344, + "kl": 2.47265625, + "learning_rate": 8.735539086499291e-07, + "loss": 0.2062, + "num_tokens": 563814917.0, + "reward": 1.0048828125, + "reward_std": 0.25415509939193726, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.9384765625, + "rewards/tag_count_reward/std": 0.1731034219264984, + "step": 938 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1952.0, + "completions/mean_length": 788.453125, + "completions/mean_terminated_length": 760.7984008789062, + "completions/min_length": 131.0, + "completions/min_terminated_length": 131.0, + "epoch": 0.32055987027396093, + "grad_norm": 6.041194438934326, + "kl": 2.4609375, + "learning_rate": 8.731810835098805e-07, + "loss": 0.1909, + "num_tokens": 564293165.0, + "reward": 0.98046875, + "reward_std": 0.2068929672241211, + "rewards/accuracy_reward/mean": 0.03125, + "rewards/accuracy_reward/std": 0.17416280508041382, + "rewards/format_reward/mean": 0.0078125, + "rewards/format_reward/std": 0.08812850713729858, + "rewards/tag_count_reward/mean": 0.94140625, + "rewards/tag_count_reward/std": 0.15945225954055786, + "step": 939 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.005859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2014.0, + "completions/mean_length": 729.88671875, + "completions/mean_terminated_length": 722.117919921875, + "completions/min_length": 9.0, + "completions/min_terminated_length": 9.0, + "epoch": 0.3209012545873517, + "grad_norm": 5.101868629455566, + "kl": 2.65234375, + "learning_rate": 8.728077996733945e-07, + "loss": 0.2009, + "num_tokens": 564740915.0, + "reward": 1.0068359375, + "reward_std": 0.21976228058338165, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.9462890625, + "rewards/tag_count_reward/std": 0.150978222489357, + "step": 940 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 776.39453125, + "completions/mean_terminated_length": 751.0637817382812, + "completions/min_length": 34.0, + "completions/min_terminated_length": 34.0, + "epoch": 0.3212426389007425, + "grad_norm": 2.568970203399658, + "kl": 3.15625, + "learning_rate": 8.724340576702791e-07, + "loss": 0.2346, + "num_tokens": 565213661.0, + "reward": 1.02392578125, + "reward_std": 0.2454395741224289, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.94189453125, + "rewards/tag_count_reward/std": 0.17037923634052277, + "step": 941 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 860.27734375, + "completions/mean_terminated_length": 824.4305419921875, + "completions/min_length": 147.0, + "completions/min_terminated_length": 147.0, + "epoch": 0.3215840232141333, + "grad_norm": 3.625922441482544, + "kl": 4.6875, + "learning_rate": 8.72059858030993e-07, + "loss": 0.2448, + "num_tokens": 565751115.0, + "reward": 0.9619140625, + "reward_std": 0.23727825284004211, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.9169921875, + "rewards/tag_count_reward/std": 0.1926850527524948, + "step": 942 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1892.0, + "completions/mean_length": 714.87109375, + "completions/mean_terminated_length": 699.0632934570312, + "completions/min_length": 40.0, + "completions/min_terminated_length": 40.0, + "epoch": 0.32192540752752413, + "grad_norm": 4.811487674713135, + "kl": 4.54296875, + "learning_rate": 8.716852012866438e-07, + "loss": 0.2542, + "num_tokens": 566192329.0, + "reward": 1.05224609375, + "reward_std": 0.2621305286884308, + "rewards/accuracy_reward/mean": 0.107421875, + "rewards/accuracy_reward/std": 0.30995169281959534, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.93896484375, + "rewards/tag_count_reward/std": 0.16196657717227936, + "step": 943 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.009765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 747.833984375, + "completions/mean_terminated_length": 735.0118408203125, + "completions/min_length": 102.0, + "completions/min_terminated_length": 102.0, + "epoch": 0.3222667918409149, + "grad_norm": 3.6192831993103027, + "kl": 4.82421875, + "learning_rate": 8.713100879689886e-07, + "loss": 0.2576, + "num_tokens": 566651476.0, + "reward": 1.0556640625, + "reward_std": 0.27593910694122314, + "rewards/accuracy_reward/mean": 0.111328125, + "rewards/accuracy_reward/std": 0.31484565138816833, + "rewards/format_reward/mean": 0.009765625, + "rewards/format_reward/std": 0.09843364357948303, + "rewards/tag_count_reward/mean": 0.9345703125, + "rewards/tag_count_reward/std": 0.17588526010513306, + "step": 944 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1891.0, + "completions/mean_length": 726.2421875, + "completions/mean_terminated_length": 697.2215576171875, + "completions/min_length": 15.0, + "completions/min_terminated_length": 15.0, + "epoch": 0.3226081761543057, + "grad_norm": 4.048362731933594, + "kl": 3.89453125, + "learning_rate": 8.709345186104319e-07, + "loss": 0.1823, + "num_tokens": 567098240.0, + "reward": 1.01416015625, + "reward_std": 0.26617729663848877, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.0078125, + "rewards/format_reward/std": 0.08812850713729858, + "rewards/tag_count_reward/mean": 0.93212890625, + "rewards/tag_count_reward/std": 0.17600135505199432, + "step": 945 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1872.0, + "completions/mean_length": 702.59375, + "completions/mean_terminated_length": 692.0, + "completions/min_length": 83.0, + "completions/min_terminated_length": 83.0, + "epoch": 0.3229495604676965, + "grad_norm": 3.2535111904144287, + "kl": 3.25, + "learning_rate": 8.705584937440257e-07, + "loss": 0.1517, + "num_tokens": 567544672.0, + "reward": 1.06005859375, + "reward_std": 0.23062613606452942, + "rewards/accuracy_reward/mean": 0.109375, + "rewards/accuracy_reward/std": 0.31241437792778015, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.94482421875, + "rewards/tag_count_reward/std": 0.15406294167041779, + "step": 946 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.001953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1906.0, + "completions/mean_length": 737.283203125, + "completions/mean_terminated_length": 734.7182006835938, + "completions/min_length": 135.0, + "completions/min_terminated_length": 135.0, + "epoch": 0.32329094478108733, + "grad_norm": 1.3384392261505127, + "kl": 2.703125, + "learning_rate": 8.701820139034686e-07, + "loss": 0.1537, + "num_tokens": 568000337.0, + "reward": 1.052734375, + "reward_std": 0.22163353860378265, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.001953125, + "rewards/format_reward/std": 0.04419417306780815, + "rewards/tag_count_reward/mean": 0.9453125, + "rewards/tag_count_reward/std": 0.14899368584156036, + "step": 947 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.009765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1867.0, + "completions/mean_length": 777.642578125, + "completions/mean_terminated_length": 765.1143798828125, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "epoch": 0.3236323290944781, + "grad_norm": 1.0986766815185547, + "kl": 1.90625, + "learning_rate": 8.698050796231049e-07, + "loss": 0.1157, + "num_tokens": 568467706.0, + "reward": 1.04443359375, + "reward_std": 0.20831140875816345, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.95654296875, + "rewards/tag_count_reward/std": 0.13613753020763397, + "step": 948 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1897.0, + "completions/mean_length": 773.7734375, + "completions/mean_terminated_length": 758.6640625, + "completions/min_length": 204.0, + "completions/min_terminated_length": 204.0, + "epoch": 0.3239737134078689, + "grad_norm": 3.154078245162964, + "kl": 1.720703125, + "learning_rate": 8.694276914379237e-07, + "loss": 0.1252, + "num_tokens": 568940246.0, + "reward": 1.04638671875, + "reward_std": 0.261174738407135, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.009765625, + "rewards/format_reward/std": 0.09843364357948303, + "rewards/tag_count_reward/mean": 0.94677734375, + "rewards/tag_count_reward/std": 0.1491146832704544, + "step": 949 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1977.0, + "completions/mean_length": 736.962890625, + "completions/mean_terminated_length": 726.6397705078125, + "completions/min_length": 152.0, + "completions/min_terminated_length": 152.0, + "epoch": 0.3243150977212597, + "grad_norm": 1.677687644958496, + "kl": 1.6796875, + "learning_rate": 8.690498498835586e-07, + "loss": 0.107, + "num_tokens": 569403843.0, + "reward": 1.06494140625, + "reward_std": 0.25524967908859253, + "rewards/accuracy_reward/mean": 0.111328125, + "rewards/accuracy_reward/std": 0.31484565138816833, + "rewards/format_reward/mean": 0.009765625, + "rewards/format_reward/std": 0.09843364357948303, + "rewards/tag_count_reward/mean": 0.94384765625, + "rewards/tag_count_reward/std": 0.15841138362884521, + "step": 950 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2000.0, + "completions/mean_length": 783.171875, + "completions/mean_terminated_length": 773.2125854492188, + "completions/min_length": 230.0, + "completions/min_terminated_length": 230.0, + "epoch": 0.32465648203465053, + "grad_norm": 2.603933334350586, + "kl": 2.146484375, + "learning_rate": 8.686715554962869e-07, + "loss": 0.1315, + "num_tokens": 569888747.0, + "reward": 1.04443359375, + "reward_std": 0.2933083176612854, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.93310546875, + "rewards/tag_count_reward/std": 0.17145265638828278, + "step": 951 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1972.0, + "completions/mean_length": 757.67578125, + "completions/mean_terminated_length": 742.3755493164062, + "completions/min_length": 203.0, + "completions/min_terminated_length": 203.0, + "epoch": 0.3249978663480413, + "grad_norm": 1.5893560647964478, + "kl": 3.08984375, + "learning_rate": 8.682928088130278e-07, + "loss": 0.1757, + "num_tokens": 570356213.0, + "reward": 1.04296875, + "reward_std": 0.29115957021713257, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.0078125, + "rewards/format_reward/std": 0.08812850713729858, + "rewards/tag_count_reward/mean": 0.9296875, + "rewards/tag_count_reward/std": 0.17468871176242828, + "step": 952 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 826.5234375, + "completions/mean_terminated_length": 787.1209716796875, + "completions/min_length": 4.0, + "completions/min_terminated_length": 4.0, + "epoch": 0.3253392506614321, + "grad_norm": 3.879516839981079, + "kl": 5.0859375, + "learning_rate": 8.679136103713431e-07, + "loss": 0.277, + "num_tokens": 570859249.0, + "reward": 0.9921875, + "reward_std": 0.2740318775177002, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.90625, + "rewards/tag_count_reward/std": 0.19659529626369476, + "step": 953 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2008.0, + "completions/mean_length": 789.013671875, + "completions/mean_terminated_length": 774.0850219726562, + "completions/min_length": 133.0, + "completions/min_terminated_length": 133.0, + "epoch": 0.3256806349748229, + "grad_norm": 2.807542085647583, + "kl": 3.13671875, + "learning_rate": 8.675339607094356e-07, + "loss": 0.148, + "num_tokens": 571337144.0, + "reward": 1.01171875, + "reward_std": 0.2758334279060364, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.017578125, + "rewards/format_reward/std": 0.13154059648513794, + "rewards/tag_count_reward/mean": 0.9296875, + "rewards/tag_count_reward/std": 0.17043600976467133, + "step": 954 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 771.439453125, + "completions/mean_terminated_length": 751.1766357421875, + "completions/min_length": 139.0, + "completions/min_terminated_length": 139.0, + "epoch": 0.3260220192882137, + "grad_norm": 1.9244072437286377, + "kl": 3.5625, + "learning_rate": 8.671538603661489e-07, + "loss": 0.1791, + "num_tokens": 571805017.0, + "reward": 1.056640625, + "reward_std": 0.3241751194000244, + "rewards/accuracy_reward/mean": 0.107421875, + "rewards/accuracy_reward/std": 0.30995169281959534, + "rewards/format_reward/mean": 0.01953125, + "rewards/format_reward/std": 0.1385180652141571, + "rewards/tag_count_reward/mean": 0.9296875, + "rewards/tag_count_reward/std": 0.17468871176242828, + "step": 955 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 781.017578125, + "completions/mean_terminated_length": 753.1995849609375, + "completions/min_length": 8.0, + "completions/min_terminated_length": 8.0, + "epoch": 0.3263634036016045, + "grad_norm": 2.9450836181640625, + "kl": 4.9140625, + "learning_rate": 8.667733098809655e-07, + "loss": 0.2572, + "num_tokens": 572278274.0, + "reward": 1.05126953125, + "reward_std": 0.36355161666870117, + "rewards/accuracy_reward/mean": 0.123046875, + "rewards/accuracy_reward/std": 0.32881227135658264, + "rewards/format_reward/mean": 0.01953125, + "rewards/format_reward/std": 0.1385180652141571, + "rewards/tag_count_reward/mean": 0.90869140625, + "rewards/tag_count_reward/std": 0.2047327607870102, + "step": 956 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2003.0, + "completions/mean_length": 801.27734375, + "completions/mean_terminated_length": 778.9701538085938, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "epoch": 0.3267047879149953, + "grad_norm": 4.156304359436035, + "kl": 3.5703125, + "learning_rate": 8.663923097940072e-07, + "loss": 0.2343, + "num_tokens": 572765744.0, + "reward": 1.00732421875, + "reward_std": 0.303446888923645, + "rewards/accuracy_reward/mean": 0.08467742055654526, + "rewards/accuracy_reward/std": 0.278682142496109, + "rewards/format_reward/mean": 0.01171875, + "rewards/format_reward/std": 0.10772226005792618, + "rewards/tag_count_reward/mean": 0.91357421875, + "rewards/tag_count_reward/std": 0.20565944910049438, + "step": 957 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1906.0, + "completions/mean_length": 783.15234375, + "completions/mean_terminated_length": 755.3812255859375, + "completions/min_length": 43.0, + "completions/min_terminated_length": 43.0, + "epoch": 0.3270461722283861, + "grad_norm": 2.296147584915161, + "kl": 3.46875, + "learning_rate": 8.660108606460343e-07, + "loss": 0.1805, + "num_tokens": 573242446.0, + "reward": 0.9892578125, + "reward_std": 0.2723425030708313, + "rewards/accuracy_reward/mean": 0.05645161122083664, + "rewards/accuracy_reward/std": 0.23102475702762604, + "rewards/format_reward/mean": 0.015625, + "rewards/format_reward/std": 0.12414088100194931, + "rewards/tag_count_reward/mean": 0.9189453125, + "rewards/tag_count_reward/std": 0.19351638853549957, + "step": 958 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1982.0, + "completions/mean_length": 768.36328125, + "completions/mean_terminated_length": 737.6520385742188, + "completions/min_length": 18.0, + "completions/min_terminated_length": 18.0, + "epoch": 0.3273875565417769, + "grad_norm": 1.4120590686798096, + "kl": 2.927734375, + "learning_rate": 8.656289629784439e-07, + "loss": 0.181, + "num_tokens": 573708504.0, + "reward": 1.03125, + "reward_std": 0.25113603472709656, + "rewards/accuracy_reward/mean": 0.087890625, + "rewards/accuracy_reward/std": 0.2834126651287079, + "rewards/format_reward/mean": 0.01171875, + "rewards/format_reward/std": 0.10772226005792618, + "rewards/tag_count_reward/mean": 0.931640625, + "rewards/tag_count_reward/std": 0.17051447927951813, + "step": 959 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1833.0, + "completions/mean_length": 777.876953125, + "completions/mean_terminated_length": 757.71630859375, + "completions/min_length": 11.0, + "completions/min_terminated_length": 11.0, + "epoch": 0.3277289408551677, + "grad_norm": 1.5171974897384644, + "kl": 2.580078125, + "learning_rate": 8.652466173332698e-07, + "loss": 0.119, + "num_tokens": 574183897.0, + "reward": 1.07568359375, + "reward_std": 0.29082536697387695, + "rewards/accuracy_reward/mean": 0.125, + "rewards/accuracy_reward/std": 0.3310423493385315, + "rewards/format_reward/mean": 0.01171875, + "rewards/format_reward/std": 0.10772226005792618, + "rewards/tag_count_reward/mean": 0.93896484375, + "rewards/tag_count_reward/std": 0.15968506038188934, + "step": 960 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2005.0, + "completions/mean_length": 846.44921875, + "completions/mean_terminated_length": 812.670654296875, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "epoch": 0.3280703251685585, + "grad_norm": 4.278796672821045, + "kl": 3.5703125, + "learning_rate": 8.648638242531817e-07, + "loss": 0.2517, + "num_tokens": 574695887.0, + "reward": 1.046875, + "reward_std": 0.28213781118392944, + "rewards/accuracy_reward/mean": 0.11328125, + "rewards/accuracy_reward/std": 0.3172462284564972, + "rewards/format_reward/mean": 0.0078125, + "rewards/format_reward/std": 0.08812850713729858, + "rewards/tag_count_reward/mean": 0.92578125, + "rewards/tag_count_reward/std": 0.1840227097272873, + "step": 961 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1910.0, + "completions/mean_length": 918.970703125, + "completions/mean_terminated_length": 877.83203125, + "completions/min_length": 152.0, + "completions/min_terminated_length": 152.0, + "epoch": 0.3284117094819493, + "grad_norm": 3.791576623916626, + "kl": 3.14453125, + "learning_rate": 8.644805842814846e-07, + "loss": 0.2258, + "num_tokens": 575248672.0, + "reward": 0.97607421875, + "reward_std": 0.2750016450881958, + "rewards/accuracy_reward/mean": 0.04838709533214569, + "rewards/accuracy_reward/std": 0.21479946374893188, + "rewards/format_reward/mean": 0.009765625, + "rewards/format_reward/std": 0.09843364357948303, + "rewards/tag_count_reward/mean": 0.91943359375, + "rewards/tag_count_reward/std": 0.1965412050485611, + "step": 962 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2022.0, + "completions/mean_length": 833.83984375, + "completions/mean_terminated_length": 812.1152954101562, + "completions/min_length": 192.0, + "completions/min_terminated_length": 192.0, + "epoch": 0.3287530937953401, + "grad_norm": 2.048088550567627, + "kl": 3.6796875, + "learning_rate": 8.640968979621174e-07, + "loss": 0.2305, + "num_tokens": 575758190.0, + "reward": 1.01220703125, + "reward_std": 0.2654229402542114, + "rewards/accuracy_reward/mean": 0.07258064299821854, + "rewards/accuracy_reward/std": 0.25970885157585144, + "rewards/format_reward/mean": 0.009765625, + "rewards/format_reward/std": 0.09843364357948303, + "rewards/tag_count_reward/mean": 0.93212890625, + "rewards/tag_count_reward/std": 0.17530503869056702, + "step": 963 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1822.0, + "completions/mean_length": 829.357421875, + "completions/mean_terminated_length": 790.0463256835938, + "completions/min_length": 12.0, + "completions/min_terminated_length": 12.0, + "epoch": 0.3290944781087309, + "grad_norm": 1.7086501121520996, + "kl": 4.04296875, + "learning_rate": 8.637127658396526e-07, + "loss": 0.2484, + "num_tokens": 576260197.0, + "reward": 1.0205078125, + "reward_std": 0.276422917842865, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.9306640625, + "rewards/tag_count_reward/std": 0.18193122744560242, + "step": 964 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 882.51953125, + "completions/mean_terminated_length": 832.672119140625, + "completions/min_length": 174.0, + "completions/min_terminated_length": 174.0, + "epoch": 0.3294358624221217, + "grad_norm": 1.9281989336013794, + "kl": 4.921875, + "learning_rate": 8.633281884592957e-07, + "loss": 0.3391, + "num_tokens": 576790463.0, + "reward": 0.98095703125, + "reward_std": 0.24103042483329773, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.009765625, + "rewards/format_reward/std": 0.09843364357948303, + "rewards/tag_count_reward/mean": 0.92236328125, + "rewards/tag_count_reward/std": 0.1815967559814453, + "step": 965 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1955.0, + "completions/mean_length": 848.255859375, + "completions/mean_terminated_length": 807.0525512695312, + "completions/min_length": 82.0, + "completions/min_terminated_length": 82.0, + "epoch": 0.3297772467355125, + "grad_norm": 3.4018478393554688, + "kl": 4.98828125, + "learning_rate": 8.629431663668834e-07, + "loss": 0.3018, + "num_tokens": 577315090.0, + "reward": 0.9697265625, + "reward_std": 0.24910764396190643, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.00390625, + "rewards/format_reward/std": 0.06243881583213806, + "rewards/tag_count_reward/mean": 0.9130859375, + "rewards/tag_count_reward/std": 0.19538374245166779, + "step": 966 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1974.0, + "completions/mean_length": 775.2421875, + "completions/mean_terminated_length": 747.2974243164062, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "epoch": 0.3301186310489033, + "grad_norm": 3.637774705886841, + "kl": 4.91015625, + "learning_rate": 8.625577001088848e-07, + "loss": 0.2741, + "num_tokens": 577790062.0, + "reward": 0.96728515625, + "reward_std": 0.24410133063793182, + "rewards/accuracy_reward/mean": 0.04032257944345474, + "rewards/accuracy_reward/std": 0.19691328704357147, + "rewards/format_reward/mean": 0.0078125, + "rewards/format_reward/std": 0.08812850713729858, + "rewards/tag_count_reward/mean": 0.92041015625, + "rewards/tag_count_reward/std": 0.18542389571666718, + "step": 967 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2000.0, + "completions/mean_length": 789.330078125, + "completions/mean_terminated_length": 751.342041015625, + "completions/min_length": 197.0, + "completions/min_terminated_length": 197.0, + "epoch": 0.3304600153622941, + "grad_norm": 7.181121349334717, + "kl": 5.7734375, + "learning_rate": 8.621717902323987e-07, + "loss": 0.3185, + "num_tokens": 578268343.0, + "reward": 1.02001953125, + "reward_std": 0.29213935136795044, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.013671875, + "rewards/format_reward/std": 0.1162383034825325, + "rewards/tag_count_reward/mean": 0.92236328125, + "rewards/tag_count_reward/std": 0.18625172972679138, + "step": 968 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.009765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2023.0, + "completions/mean_length": 758.626953125, + "completions/mean_terminated_length": 745.9112548828125, + "completions/min_length": 130.0, + "completions/min_terminated_length": 130.0, + "epoch": 0.3308013996756849, + "grad_norm": 3.7928194999694824, + "kl": 4.27734375, + "learning_rate": 8.61785437285153e-07, + "loss": 0.2576, + "num_tokens": 578731752.0, + "reward": 1.02783203125, + "reward_std": 0.282349556684494, + "rewards/accuracy_reward/mean": 0.09879032522439957, + "rewards/accuracy_reward/std": 0.2986815273761749, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.92626953125, + "rewards/tag_count_reward/std": 0.16716337203979492, + "step": 969 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2006.0, + "completions/mean_length": 755.302734375, + "completions/mean_terminated_length": 737.3841552734375, + "completions/min_length": 119.0, + "completions/min_terminated_length": 119.0, + "epoch": 0.3311427839890757, + "grad_norm": 2.746143341064453, + "kl": 3.6484375, + "learning_rate": 8.613986418155055e-07, + "loss": 0.1963, + "num_tokens": 579199811.0, + "reward": 1.00537109375, + "reward_std": 0.24694019556045532, + "rewards/accuracy_reward/mean": 0.07258064299821854, + "rewards/accuracy_reward/std": 0.25970885157585144, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.92919921875, + "rewards/tag_count_reward/std": 0.1755392700433731, + "step": 970 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2014.0, + "completions/mean_length": 852.048828125, + "completions/mean_terminated_length": 795.7975463867188, + "completions/min_length": 56.0, + "completions/min_terminated_length": 56.0, + "epoch": 0.3314841683024665, + "grad_norm": 4.72691535949707, + "kl": 2.98828125, + "learning_rate": 8.610114043724416e-07, + "loss": 0.2596, + "num_tokens": 579727212.0, + "reward": 0.97314453125, + "reward_std": 0.25940871238708496, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.91845703125, + "rewards/tag_count_reward/std": 0.1923593431711197, + "step": 971 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1968.0, + "completions/mean_length": 878.787109375, + "completions/mean_terminated_length": 826.2918090820312, + "completions/min_length": 186.0, + "completions/min_terminated_length": 186.0, + "epoch": 0.3318255526158573, + "grad_norm": 9.027946472167969, + "kl": 3.7265625, + "learning_rate": 8.606237255055738e-07, + "loss": 0.3199, + "num_tokens": 580252207.0, + "reward": 0.99462890625, + "reward_std": 0.2623327970504761, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.92236328125, + "rewards/tag_count_reward/std": 0.18821148574352264, + "step": 972 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2021.0, + "completions/mean_length": 856.70703125, + "completions/mean_terminated_length": 800.6748046875, + "completions/min_length": 161.0, + "completions/min_terminated_length": 161.0, + "epoch": 0.3321669369292481, + "grad_norm": 5.184525012969971, + "kl": 3.69140625, + "learning_rate": 8.602356057651416e-07, + "loss": 0.2952, + "num_tokens": 580769081.0, + "reward": 1.0419921875, + "reward_std": 0.31056830286979675, + "rewards/accuracy_reward/mean": 0.111328125, + "rewards/accuracy_reward/std": 0.31484565138816833, + "rewards/format_reward/mean": 0.0078125, + "rewards/format_reward/std": 0.08812850713729858, + "rewards/tag_count_reward/mean": 0.9228515625, + "rewards/tag_count_reward/std": 0.19321990013122559, + "step": 973 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2000.0, + "completions/mean_length": 853.109375, + "completions/mean_terminated_length": 819.51806640625, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "epoch": 0.3325083212426389, + "grad_norm": 5.471117973327637, + "kl": 3.291015625, + "learning_rate": 8.598470457020101e-07, + "loss": 0.2522, + "num_tokens": 581289521.0, + "reward": 0.94287109375, + "reward_std": 0.1962069571018219, + "rewards/accuracy_reward/mean": 0.01171875, + "rewards/accuracy_reward/std": 0.10772226005792618, + "rewards/format_reward/mean": 0.005859375, + "rewards/format_reward/std": 0.07639661431312561, + "rewards/tag_count_reward/mean": 0.92529296875, + "rewards/tag_count_reward/std": 0.18349166214466095, + "step": 974 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 789.197265625, + "completions/mean_terminated_length": 761.5588989257812, + "completions/min_length": 128.0, + "completions/min_terminated_length": 128.0, + "epoch": 0.3328497055560297, + "grad_norm": 3.1377015113830566, + "kl": 4.31640625, + "learning_rate": 8.594580458676688e-07, + "loss": 0.3071, + "num_tokens": 581767174.0, + "reward": 1.033203125, + "reward_std": 0.2888753414154053, + "rewards/accuracy_reward/mean": 0.099609375, + "rewards/accuracy_reward/std": 0.29977133870124817, + "rewards/format_reward/mean": 0.0078125, + "rewards/format_reward/std": 0.08812850713729858, + "rewards/tag_count_reward/mean": 0.92578125, + "rewards/tag_count_reward/std": 0.18796826899051666, + "step": 975 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 867.1953125, + "completions/mean_terminated_length": 804.024658203125, + "completions/min_length": 162.0, + "completions/min_terminated_length": 162.0, + "epoch": 0.3331910898694205, + "grad_norm": 6.757725715637207, + "kl": 6.0546875, + "learning_rate": 8.59068606814232e-07, + "loss": 0.3132, + "num_tokens": 582291642.0, + "reward": 1.03955078125, + "reward_std": 0.30651384592056274, + "rewards/accuracy_reward/mean": 0.1171875, + "rewards/accuracy_reward/std": 0.32195815443992615, + "rewards/format_reward/mean": 0.01953125, + "rewards/format_reward/std": 0.1385180652141571, + "rewards/tag_count_reward/mean": 0.90283203125, + "rewards/tag_count_reward/std": 0.2044198215007782, + "step": 976 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2014.0, + "completions/mean_length": 893.42578125, + "completions/mean_terminated_length": 848.9290161132812, + "completions/min_length": 98.0, + "completions/min_terminated_length": 98.0, + "epoch": 0.3335324741828113, + "grad_norm": 4.673368453979492, + "kl": 7.1875, + "learning_rate": 8.586787290944373e-07, + "loss": 0.4068, + "num_tokens": 582821284.0, + "reward": 1.064453125, + "reward_std": 0.33829671144485474, + "rewards/accuracy_reward/mean": 0.150390625, + "rewards/accuracy_reward/std": 0.35780346393585205, + "rewards/format_reward/mean": 0.013671875, + "rewards/format_reward/std": 0.1162383034825325, + "rewards/tag_count_reward/mean": 0.900390625, + "rewards/tag_count_reward/std": 0.21579019725322723, + "step": 977 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 807.15234375, + "completions/mean_terminated_length": 748.7893676757812, + "completions/min_length": 82.0, + "completions/min_terminated_length": 82.0, + "epoch": 0.3338738584962021, + "grad_norm": 6.821012020111084, + "kl": 6.984375, + "learning_rate": 8.582884132616448e-07, + "loss": 0.3878, + "num_tokens": 583314690.0, + "reward": 0.9775390625, + "reward_std": 0.3012595772743225, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.02734375, + "rewards/format_reward/std": 0.16324250400066376, + "rewards/tag_count_reward/mean": 0.8994140625, + "rewards/tag_count_reward/std": 0.21871723234653473, + "step": 978 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1975.0, + "completions/mean_length": 824.62890625, + "completions/mean_terminated_length": 767.087890625, + "completions/min_length": 145.0, + "completions/min_terminated_length": 145.0, + "epoch": 0.3342152428095929, + "grad_norm": 2.729808807373047, + "kl": 7.046875, + "learning_rate": 8.578976598698364e-07, + "loss": 0.4281, + "num_tokens": 583807972.0, + "reward": 1.00146484375, + "reward_std": 0.33582258224487305, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.0234375, + "rewards/format_reward/std": 0.15143637359142303, + "rewards/tag_count_reward/mean": 0.89599609375, + "rewards/tag_count_reward/std": 0.21851344406604767, + "step": 979 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2028.0, + "completions/mean_length": 871.8984375, + "completions/mean_terminated_length": 806.4247436523438, + "completions/min_length": 126.0, + "completions/min_terminated_length": 126.0, + "epoch": 0.3345566271229837, + "grad_norm": 4.520415306091309, + "kl": 7.375, + "learning_rate": 8.57506469473615e-07, + "loss": 0.4489, + "num_tokens": 584326240.0, + "reward": 0.94970703125, + "reward_std": 0.31445467472076416, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.01953125, + "rewards/format_reward/std": 0.1385180652141571, + "rewards/tag_count_reward/mean": 0.88525390625, + "rewards/tag_count_reward/std": 0.226416677236557, + "step": 980 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 902.8515625, + "completions/mean_terminated_length": 851.4367065429688, + "completions/min_length": 187.0, + "completions/min_terminated_length": 187.0, + "epoch": 0.3348980114363745, + "grad_norm": 2.7762203216552734, + "kl": 5.7890625, + "learning_rate": 8.57114842628204e-07, + "loss": 0.3659, + "num_tokens": 584862580.0, + "reward": 1.01416015625, + "reward_std": 0.3422966003417969, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.02734375, + "rewards/format_reward/std": 0.16324250400066376, + "rewards/tag_count_reward/mean": 0.90673828125, + "rewards/tag_count_reward/std": 0.20741750299930573, + "step": 981 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2033.0, + "completions/mean_length": 869.392578125, + "completions/mean_terminated_length": 801.2086181640625, + "completions/min_length": 129.0, + "completions/min_terminated_length": 129.0, + "epoch": 0.3352393957497653, + "grad_norm": 3.293605327606201, + "kl": 4.46875, + "learning_rate": 8.567227798894458e-07, + "loss": 0.2956, + "num_tokens": 585385453.0, + "reward": 0.99560546875, + "reward_std": 0.32971763610839844, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.03515625, + "rewards/format_reward/std": 0.1843547374010086, + "rewards/tag_count_reward/mean": 0.89599609375, + "rewards/tag_count_reward/std": 0.21963004767894745, + "step": 982 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2016.0, + "completions/mean_length": 908.6015625, + "completions/mean_terminated_length": 827.5564575195312, + "completions/min_length": 127.0, + "completions/min_terminated_length": 127.0, + "epoch": 0.3355807800631561, + "grad_norm": 9.126565933227539, + "kl": 4.265625, + "learning_rate": 8.56330281813802e-07, + "loss": 0.3588, + "num_tokens": 585937281.0, + "reward": 1.03515625, + "reward_std": 0.31757861375808716, + "rewards/accuracy_reward/mean": 0.119140625, + "rewards/accuracy_reward/std": 0.32427072525024414, + "rewards/format_reward/mean": 0.01953125, + "rewards/format_reward/std": 0.1385180652141571, + "rewards/tag_count_reward/mean": 0.896484375, + "rewards/tag_count_reward/std": 0.2173432856798172, + "step": 983 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.060546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1996.0, + "completions/mean_length": 863.005859375, + "completions/mean_terminated_length": 786.6340942382812, + "completions/min_length": 22.0, + "completions/min_terminated_length": 22.0, + "epoch": 0.3359221643765469, + "grad_norm": 6.865476131439209, + "kl": 3.37890625, + "learning_rate": 8.559373489583518e-07, + "loss": 0.28, + "num_tokens": 586456100.0, + "reward": 1.07373046875, + "reward_std": 0.38225170969963074, + "rewards/accuracy_reward/mean": 0.140625, + "rewards/accuracy_reward/std": 0.3479743003845215, + "rewards/format_reward/mean": 0.03125, + "rewards/format_reward/std": 0.17416280508041382, + "rewards/tag_count_reward/mean": 0.90185546875, + "rewards/tag_count_reward/std": 0.21674399077892303, + "step": 984 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.072265625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 924.205078125, + "completions/mean_terminated_length": 836.6673583984375, + "completions/min_length": 38.0, + "completions/min_terminated_length": 38.0, + "epoch": 0.3362635486899377, + "grad_norm": 7.508963108062744, + "kl": 3.328125, + "learning_rate": 8.555439818807914e-07, + "loss": 0.3158, + "num_tokens": 587018125.0, + "reward": 1.04296875, + "reward_std": 0.34603816270828247, + "rewards/accuracy_reward/mean": 0.126953125, + "rewards/accuracy_reward/std": 0.33324605226516724, + "rewards/format_reward/mean": 0.013671875, + "rewards/format_reward/std": 0.1162383034825325, + "rewards/tag_count_reward/mean": 0.90234375, + "rewards/tag_count_reward/std": 0.21326914429664612, + "step": 985 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1980.0, + "completions/mean_length": 914.611328125, + "completions/mean_terminated_length": 858.870849609375, + "completions/min_length": 210.0, + "completions/min_terminated_length": 210.0, + "epoch": 0.3366049330033285, + "grad_norm": 8.431133270263672, + "kl": 3.2578125, + "learning_rate": 8.551501811394335e-07, + "loss": 0.2678, + "num_tokens": 587567622.0, + "reward": 0.95947265625, + "reward_std": 0.285304456949234, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.015625, + "rewards/format_reward/std": 0.12414088100194931, + "rewards/tag_count_reward/mean": 0.90478515625, + "rewards/tag_count_reward/std": 0.20770753920078278, + "step": 986 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1981.0, + "completions/mean_length": 822.861328125, + "completions/mean_terminated_length": 775.64501953125, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.3369463173167193, + "grad_norm": 4.268903732299805, + "kl": 3.21484375, + "learning_rate": 8.547559472932062e-07, + "loss": 0.2195, + "num_tokens": 588068351.0, + "reward": 1.00244140625, + "reward_std": 0.27478182315826416, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.01953125, + "rewards/format_reward/std": 0.1385180652141571, + "rewards/tag_count_reward/mean": 0.91650390625, + "rewards/tag_count_reward/std": 0.19841860234737396, + "step": 987 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1972.0, + "completions/mean_length": 869.857421875, + "completions/mean_terminated_length": 836.7369384765625, + "completions/min_length": 4.0, + "completions/min_terminated_length": 4.0, + "epoch": 0.33728770163011007, + "grad_norm": 1.8480836153030396, + "kl": 3.3671875, + "learning_rate": 8.543612809016524e-07, + "loss": 0.2163, + "num_tokens": 588590518.0, + "reward": 0.9873046875, + "reward_std": 0.24880146980285645, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.015625, + "rewards/format_reward/std": 0.12414088100194931, + "rewards/tag_count_reward/mean": 0.9248046875, + "rewards/tag_count_reward/std": 0.18428993225097656, + "step": 988 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2005.0, + "completions/mean_length": 832.720703125, + "completions/mean_terminated_length": 806.0379028320312, + "completions/min_length": 155.0, + "completions/min_terminated_length": 155.0, + "epoch": 0.3376290859435009, + "grad_norm": 4.212379455566406, + "kl": 4.4296875, + "learning_rate": 8.539661825249287e-07, + "loss": 0.2464, + "num_tokens": 589098391.0, + "reward": 1.0546875, + "reward_std": 0.304060697555542, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.0390625, + "rewards/format_reward/std": 0.1939331740140915, + "rewards/tag_count_reward/mean": 0.931640625, + "rewards/tag_count_reward/std": 0.1733599156141281, + "step": 989 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1922.0, + "completions/mean_length": 838.3984375, + "completions/mean_terminated_length": 806.8858032226562, + "completions/min_length": 168.0, + "completions/min_terminated_length": 168.0, + "epoch": 0.3379704702568917, + "grad_norm": 7.339483737945557, + "kl": 5.67578125, + "learning_rate": 8.535706527238051e-07, + "loss": 0.2762, + "num_tokens": 589609843.0, + "reward": 0.984375, + "reward_std": 0.28474554419517517, + "rewards/accuracy_reward/mean": 0.03515625, + "rewards/accuracy_reward/std": 0.1843547374010086, + "rewards/format_reward/mean": 0.029296875, + "rewards/format_reward/std": 0.16880230605602264, + "rewards/tag_count_reward/mean": 0.919921875, + "rewards/tag_count_reward/std": 0.19642995297908783, + "step": 990 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2005.0, + "completions/mean_length": 753.921875, + "completions/mean_terminated_length": 730.767333984375, + "completions/min_length": 174.0, + "completions/min_terminated_length": 174.0, + "epoch": 0.3383118545702825, + "grad_norm": 4.1210618019104, + "kl": 5.7265625, + "learning_rate": 8.531746920596639e-07, + "loss": 0.3426, + "num_tokens": 590078555.0, + "reward": 1.03662109375, + "reward_std": 0.3218595087528229, + "rewards/accuracy_reward/mean": 0.07258064299821854, + "rewards/accuracy_reward/std": 0.25970885157585144, + "rewards/format_reward/mean": 0.041015625, + "rewards/format_reward/std": 0.19852031767368317, + "rewards/tag_count_reward/mean": 0.92529296875, + "rewards/tag_count_reward/std": 0.1808057427406311, + "step": 991 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1951.0, + "completions/mean_length": 813.31640625, + "completions/mean_terminated_length": 791.224609375, + "completions/min_length": 258.0, + "completions/min_terminated_length": 258.0, + "epoch": 0.33865323888367327, + "grad_norm": 1.6163721084594727, + "kl": 4.328125, + "learning_rate": 8.527783010944986e-07, + "loss": 0.2373, + "num_tokens": 590577277.0, + "reward": 1.01953125, + "reward_std": 0.27651625871658325, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.015625, + "rewards/format_reward/std": 0.12414088100194931, + "rewards/tag_count_reward/mean": 0.92578125, + "rewards/tag_count_reward/std": 0.17446976900100708, + "step": 992 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1994.0, + "completions/mean_length": 802.09765625, + "completions/mean_terminated_length": 769.6392822265625, + "completions/min_length": 154.0, + "completions/min_terminated_length": 154.0, + "epoch": 0.3389946231970641, + "grad_norm": 3.7197697162628174, + "kl": 5.15625, + "learning_rate": 8.523814803909137e-07, + "loss": 0.3017, + "num_tokens": 591065839.0, + "reward": 1.0087890625, + "reward_std": 0.32167479395866394, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.03125, + "rewards/format_reward/std": 0.17416280508041382, + "rewards/tag_count_reward/mean": 0.9130859375, + "rewards/tag_count_reward/std": 0.19910427927970886, + "step": 993 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1959.0, + "completions/mean_length": 764.3671875, + "completions/mean_terminated_length": 754.2598266601562, + "completions/min_length": 192.0, + "completions/min_terminated_length": 192.0, + "epoch": 0.3393360075104549, + "grad_norm": 2.030200242996216, + "kl": 3.28125, + "learning_rate": 8.51984230512124e-07, + "loss": 0.1716, + "num_tokens": 591533739.0, + "reward": 1.0595703125, + "reward_std": 0.26806777715682983, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.021484375, + "rewards/format_reward/std": 0.14513419568538666, + "rewards/tag_count_reward/mean": 0.9423828125, + "rewards/tag_count_reward/std": 0.16057194769382477, + "step": 994 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.009765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1950.0, + "completions/mean_length": 780.6484375, + "completions/mean_terminated_length": 768.14990234375, + "completions/min_length": 143.0, + "completions/min_terminated_length": 143.0, + "epoch": 0.3396773918238457, + "grad_norm": 1.6519471406936646, + "kl": 2.455078125, + "learning_rate": 8.515865520219526e-07, + "loss": 0.1544, + "num_tokens": 592010247.0, + "reward": 1.01318359375, + "reward_std": 0.2568773627281189, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.021484375, + "rewards/format_reward/std": 0.14513419568538666, + "rewards/tag_count_reward/mean": 0.93115234375, + "rewards/tag_count_reward/std": 0.16995809972286224, + "step": 995 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 792.646484375, + "completions/mean_terminated_length": 777.7609252929688, + "completions/min_length": 62.0, + "completions/min_terminated_length": 62.0, + "epoch": 0.34001877613723647, + "grad_norm": 0.7317795157432556, + "kl": 1.724609375, + "learning_rate": 8.511884454848315e-07, + "loss": 0.0839, + "num_tokens": 592490946.0, + "reward": 1.09521484375, + "reward_std": 0.31519752740859985, + "rewards/accuracy_reward/mean": 0.115234375, + "rewards/accuracy_reward/std": 0.3196168541908264, + "rewards/format_reward/mean": 0.03515625, + "rewards/format_reward/std": 0.1843547374010086, + "rewards/tag_count_reward/mean": 0.94482421875, + "rewards/tag_count_reward/std": 0.15166257321834564, + "step": 996 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1992.0, + "completions/mean_length": 770.0546875, + "completions/mean_terminated_length": 754.9012451171875, + "completions/min_length": 269.0, + "completions/min_terminated_length": 269.0, + "epoch": 0.3403601604506273, + "grad_norm": 3.0594396591186523, + "kl": 2.16796875, + "learning_rate": 8.507899114658003e-07, + "loss": 0.1328, + "num_tokens": 592963934.0, + "reward": 1.1103515625, + "reward_std": 0.331748902797699, + "rewards/accuracy_reward/mean": 0.130859375, + "rewards/accuracy_reward/std": 0.33757632970809937, + "rewards/format_reward/mean": 0.033203125, + "rewards/format_reward/std": 0.17934183776378632, + "rewards/tag_count_reward/mean": 0.9462890625, + "rewards/tag_count_reward/std": 0.15654632449150085, + "step": 997 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1949.0, + "completions/mean_length": 845.802734375, + "completions/mean_terminated_length": 809.5191040039062, + "completions/min_length": 245.0, + "completions/min_terminated_length": 245.0, + "epoch": 0.3407015447640181, + "grad_norm": 2.0634729862213135, + "kl": 2.4453125, + "learning_rate": 8.503909505305048e-07, + "loss": 0.161, + "num_tokens": 593476969.0, + "reward": 1.091796875, + "reward_std": 0.3658602237701416, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.060546875, + "rewards/format_reward/std": 0.2387305200099945, + "rewards/tag_count_reward/mean": 0.939453125, + "rewards/tag_count_reward/std": 0.16025325655937195, + "step": 998 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 774.53515625, + "completions/mean_terminated_length": 751.7494506835938, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "epoch": 0.3410429290774089, + "grad_norm": 0.7929698824882507, + "kl": 3.0, + "learning_rate": 8.499915632451975e-07, + "loss": 0.1656, + "num_tokens": 593945531.0, + "reward": 1.0576171875, + "reward_std": 0.3631839156150818, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.064453125, + "rewards/format_reward/std": 0.24579854309558868, + "rewards/tag_count_reward/mean": 0.9267578125, + "rewards/tag_count_reward/std": 0.18174204230308533, + "step": 999 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 778.240234375, + "completions/mean_terminated_length": 768.2421264648438, + "completions/min_length": 158.0, + "completions/min_terminated_length": 158.0, + "epoch": 0.34138431339079967, + "grad_norm": 3.1648752689361572, + "kl": 3.103515625, + "learning_rate": 8.495917501767352e-07, + "loss": 0.1261, + "num_tokens": 594422934.0, + "reward": 1.09228515625, + "reward_std": 0.38167354464530945, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.09765625, + "rewards/format_reward/std": 0.29713961482048035, + "rewards/tag_count_reward/mean": 0.91650390625, + "rewards/tag_count_reward/std": 0.18435926735401154, + "step": 1000 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 831.1484375, + "completions/mean_terminated_length": 799.4468994140625, + "completions/min_length": 228.0, + "completions/min_terminated_length": 228.0, + "epoch": 0.3417256977041905, + "grad_norm": 1.4682717323303223, + "kl": 4.21875, + "learning_rate": 8.491915118925798e-07, + "loss": 0.2445, + "num_tokens": 594932706.0, + "reward": 1.10595703125, + "reward_std": 0.4507877826690674, + "rewards/accuracy_reward/mean": 0.12890625, + "rewards/accuracy_reward/std": 0.33542385697364807, + "rewards/format_reward/mean": 0.0859375, + "rewards/format_reward/std": 0.28054583072662354, + "rewards/tag_count_reward/mean": 0.89111328125, + "rewards/tag_count_reward/std": 0.21555037796497345, + "step": 1001 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2020.0, + "completions/mean_length": 857.244140625, + "completions/mean_terminated_length": 823.76904296875, + "completions/min_length": 17.0, + "completions/min_terminated_length": 17.0, + "epoch": 0.3420670820175813, + "grad_norm": 1.5017461776733398, + "kl": 4.0234375, + "learning_rate": 8.48790848960796e-07, + "loss": 0.2004, + "num_tokens": 595453535.0, + "reward": 1.103515625, + "reward_std": 0.4274582862854004, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.11328125, + "rewards/format_reward/std": 0.3172462284564972, + "rewards/tag_count_reward/mean": 0.90625, + "rewards/tag_count_reward/std": 0.19906827807426453, + "step": 1002 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 814.6171875, + "completions/mean_terminated_length": 792.5486450195312, + "completions/min_length": 119.0, + "completions/min_terminated_length": 119.0, + "epoch": 0.3424084663309721, + "grad_norm": 3.0235321521759033, + "kl": 3.5625, + "learning_rate": 8.483897619500517e-07, + "loss": 0.166, + "num_tokens": 595950603.0, + "reward": 1.064453125, + "reward_std": 0.42457592487335205, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.125, + "rewards/format_reward/std": 0.3310423493385315, + "rewards/tag_count_reward/mean": 0.900390625, + "rewards/tag_count_reward/std": 0.20592933893203735, + "step": 1003 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2003.0, + "completions/mean_length": 866.642578125, + "completions/mean_terminated_length": 833.4317016601562, + "completions/min_length": 147.0, + "completions/min_terminated_length": 147.0, + "epoch": 0.3427498506443629, + "grad_norm": 1.5792306661605835, + "kl": 3.8671875, + "learning_rate": 8.479882514296165e-07, + "loss": 0.2534, + "num_tokens": 596471172.0, + "reward": 1.07763671875, + "reward_std": 0.439164936542511, + "rewards/accuracy_reward/mean": 0.052419353276491165, + "rewards/accuracy_reward/std": 0.22309617698192596, + "rewards/format_reward/mean": 0.130859375, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.89599609375, + "rewards/tag_count_reward/std": 0.21053139865398407, + "step": 1004 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2004.0, + "completions/mean_length": 783.689453125, + "completions/mean_terminated_length": 758.5040283203125, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "epoch": 0.3430912349577537, + "grad_norm": 1.8494607210159302, + "kl": 2.94140625, + "learning_rate": 8.475863179693613e-07, + "loss": 0.1777, + "num_tokens": 596952453.0, + "reward": 1.283203125, + "reward_std": 0.5326836109161377, + "rewards/accuracy_reward/mean": 0.154296875, + "rewards/accuracy_reward/std": 0.36158639192581177, + "rewards/format_reward/mean": 0.2109375, + "rewards/format_reward/std": 0.4083731174468994, + "rewards/tag_count_reward/mean": 0.91796875, + "rewards/tag_count_reward/std": 0.19119404256343842, + "step": 1005 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2020.0, + "completions/mean_length": 849.076171875, + "completions/mean_terminated_length": 812.8912963867188, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "epoch": 0.3434326192711445, + "grad_norm": 1.4781209230422974, + "kl": 3.72265625, + "learning_rate": 8.471839621397569e-07, + "loss": 0.1878, + "num_tokens": 597458444.0, + "reward": 1.23876953125, + "reward_std": 0.554077684879303, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.255859375, + "rewards/format_reward/std": 0.43676990270614624, + "rewards/tag_count_reward/mean": 0.89892578125, + "rewards/tag_count_reward/std": 0.1976224035024643, + "step": 1006 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2021.0, + "completions/mean_length": 874.451171875, + "completions/mean_terminated_length": 841.4597778320312, + "completions/min_length": 194.0, + "completions/min_terminated_length": 194.0, + "epoch": 0.3437740035845353, + "grad_norm": 1.2865853309631348, + "kl": 3.65234375, + "learning_rate": 8.467811845118741e-07, + "loss": 0.1754, + "num_tokens": 597979651.0, + "reward": 1.22607421875, + "reward_std": 0.579481303691864, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.26953125, + "rewards/format_reward/std": 0.44415023922920227, + "rewards/tag_count_reward/mean": 0.89013671875, + "rewards/tag_count_reward/std": 0.2167528122663498, + "step": 1007 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 853.5625, + "completions/mean_terminated_length": 807.5294189453125, + "completions/min_length": 127.0, + "completions/min_terminated_length": 127.0, + "epoch": 0.3441153878979261, + "grad_norm": 3.341688394546509, + "kl": 4.61328125, + "learning_rate": 8.463779856573819e-07, + "loss": 0.2443, + "num_tokens": 598495827.0, + "reward": 1.30859375, + "reward_std": 0.6163730621337891, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.357421875, + "rewards/format_reward/std": 0.4797092080116272, + "rewards/tag_count_reward/mean": 0.8984375, + "rewards/tag_count_reward/std": 0.20616121590137482, + "step": 1008 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 864.275390625, + "completions/mean_terminated_length": 823.6222534179688, + "completions/min_length": 141.0, + "completions/min_terminated_length": 141.0, + "epoch": 0.3444567722113169, + "grad_norm": 6.600764751434326, + "kl": 4.6953125, + "learning_rate": 8.459743661485472e-07, + "loss": 0.2085, + "num_tokens": 599024640.0, + "reward": 1.33154296875, + "reward_std": 0.6768674254417419, + "rewards/accuracy_reward/mean": 0.060483869165182114, + "rewards/accuracy_reward/std": 0.2386218160390854, + "rewards/format_reward/mean": 0.40625, + "rewards/format_reward/std": 0.49161264300346375, + "rewards/tag_count_reward/mean": 0.86669921875, + "rewards/tag_count_reward/std": 0.22268475592136383, + "step": 1009 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2023.0, + "completions/mean_length": 736.970703125, + "completions/mean_terminated_length": 710.8546142578125, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "epoch": 0.3447981565247077, + "grad_norm": 18.189531326293945, + "kl": 5.4453125, + "learning_rate": 8.455703265582342e-07, + "loss": 0.2215, + "num_tokens": 599478081.0, + "reward": 1.39697265625, + "reward_std": 0.7271381616592407, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.447265625, + "rewards/format_reward/std": 0.4976975917816162, + "rewards/tag_count_reward/mean": 0.85595703125, + "rewards/tag_count_reward/std": 0.21869266033172607, + "step": 1010 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1998.0, + "completions/mean_length": 779.84765625, + "completions/mean_terminated_length": 754.585693359375, + "completions/min_length": 139.0, + "completions/min_terminated_length": 139.0, + "epoch": 0.3451395408380985, + "grad_norm": 24.602689743041992, + "kl": 5.59375, + "learning_rate": 8.451658674599032e-07, + "loss": 0.2229, + "num_tokens": 599952787.0, + "reward": 1.3740234375, + "reward_std": 0.6721348166465759, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.474609375, + "rewards/format_reward/std": 0.4998432695865631, + "rewards/tag_count_reward/mean": 0.8486328125, + "rewards/tag_count_reward/std": 0.21453560888767242, + "step": 1011 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1956.0, + "completions/mean_length": 766.462890625, + "completions/mean_terminated_length": 727.78466796875, + "completions/min_length": 112.0, + "completions/min_terminated_length": 112.0, + "epoch": 0.3454809251514893, + "grad_norm": 25.765657424926758, + "kl": 5.8125, + "learning_rate": 8.447609894276102e-07, + "loss": 0.2457, + "num_tokens": 600418704.0, + "reward": 1.41796875, + "reward_std": 0.6370959281921387, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.50390625, + "rewards/format_reward/std": 0.5004737377166748, + "rewards/tag_count_reward/mean": 0.87109375, + "rewards/tag_count_reward/std": 0.2008649855852127, + "step": 1012 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 822.58984375, + "completions/mean_terminated_length": 767.5714111328125, + "completions/min_length": 196.0, + "completions/min_terminated_length": 196.0, + "epoch": 0.3458223094648801, + "grad_norm": 13.013208389282227, + "kl": 4.7265625, + "learning_rate": 8.443556930360048e-07, + "loss": 0.2222, + "num_tokens": 600922270.0, + "reward": 1.52880859375, + "reward_std": 0.6884911060333252, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.57421875, + "rewards/format_reward/std": 0.4949444830417633, + "rewards/tag_count_reward/mean": 0.86865234375, + "rewards/tag_count_reward/std": 0.2149244099855423, + "step": 1013 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 790.443359375, + "completions/mean_terminated_length": 752.4888916015625, + "completions/min_length": 72.0, + "completions/min_terminated_length": 72.0, + "epoch": 0.3461636937782709, + "grad_norm": 6.748332977294922, + "kl": 4.6328125, + "learning_rate": 8.439499788603318e-07, + "loss": 0.2348, + "num_tokens": 601397009.0, + "reward": 1.5400390625, + "reward_std": 0.7158875465393066, + "rewards/accuracy_reward/mean": 0.087890625, + "rewards/accuracy_reward/std": 0.2834126651287079, + "rewards/format_reward/mean": 0.58203125, + "rewards/format_reward/std": 0.4937073290348053, + "rewards/tag_count_reward/mean": 0.8701171875, + "rewards/tag_count_reward/std": 0.21382176876068115, + "step": 1014 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 845.712890625, + "completions/mean_terminated_length": 794.291259765625, + "completions/min_length": 113.0, + "completions/min_terminated_length": 113.0, + "epoch": 0.3465050780916617, + "grad_norm": 4.612382888793945, + "kl": 3.69140625, + "learning_rate": 8.435438474764281e-07, + "loss": 0.2272, + "num_tokens": 601906414.0, + "reward": 1.58349609375, + "reward_std": 0.5909501314163208, + "rewards/accuracy_reward/mean": 0.033203125, + "rewards/accuracy_reward/std": 0.17934183776378632, + "rewards/format_reward/mean": 0.66796875, + "rewards/format_reward/std": 0.47140273451805115, + "rewards/tag_count_reward/mean": 0.88232421875, + "rewards/tag_count_reward/std": 0.20912423729896545, + "step": 1015 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1952.0, + "completions/mean_length": 837.359375, + "completions/mean_terminated_length": 803.3252563476562, + "completions/min_length": 129.0, + "completions/min_terminated_length": 129.0, + "epoch": 0.3468464624050525, + "grad_norm": 5.700664520263672, + "kl": 3.234375, + "learning_rate": 8.431372994607225e-07, + "loss": 0.1731, + "num_tokens": 602408326.0, + "reward": 1.67431640625, + "reward_std": 0.6381564140319824, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.689453125, + "rewards/format_reward/std": 0.46317005157470703, + "rewards/tag_count_reward/mean": 0.90283203125, + "rewards/tag_count_reward/std": 0.1958642452955246, + "step": 1016 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1893.0, + "completions/mean_length": 794.33984375, + "completions/mean_terminated_length": 774.4404907226562, + "completions/min_length": 25.0, + "completions/min_terminated_length": 25.0, + "epoch": 0.34718784671844327, + "grad_norm": 5.525385856628418, + "kl": 2.607421875, + "learning_rate": 8.427303353902359e-07, + "loss": 0.1343, + "num_tokens": 602895668.0, + "reward": 1.744140625, + "reward_std": 0.5869893431663513, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.7421875, + "rewards/format_reward/std": 0.43785804510116577, + "rewards/tag_count_reward/mean": 0.92578125, + "rewards/tag_count_reward/std": 0.17446976900100708, + "step": 1017 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1977.0, + "completions/mean_length": 832.45703125, + "completions/mean_terminated_length": 810.7077026367188, + "completions/min_length": 41.0, + "completions/min_terminated_length": 41.0, + "epoch": 0.3475292310318341, + "grad_norm": 2.698788642883301, + "kl": 3.30859375, + "learning_rate": 8.423229558425796e-07, + "loss": 0.1227, + "num_tokens": 603405390.0, + "reward": 1.68603515625, + "reward_std": 0.6165211200714111, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.712890625, + "rewards/format_reward/std": 0.45285552740097046, + "rewards/tag_count_reward/mean": 0.90673828125, + "rewards/tag_count_reward/std": 0.19274641573429108, + "step": 1018 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1972.0, + "completions/mean_length": 892.26953125, + "completions/mean_terminated_length": 840.3795776367188, + "completions/min_length": 78.0, + "completions/min_terminated_length": 78.0, + "epoch": 0.3478706153452249, + "grad_norm": 1.207611322402954, + "kl": 5.46875, + "learning_rate": 8.419151613959539e-07, + "loss": 0.3078, + "num_tokens": 603933416.0, + "reward": 1.6123046875, + "reward_std": 0.6389665603637695, + "rewards/accuracy_reward/mean": 0.0234375, + "rewards/accuracy_reward/std": 0.15143637359142303, + "rewards/format_reward/mean": 0.69921875, + "rewards/format_reward/std": 0.45904624462127686, + "rewards/tag_count_reward/mean": 0.8896484375, + "rewards/tag_count_reward/std": 0.21734988689422607, + "step": 1019 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2006.0, + "completions/mean_length": 810.12109375, + "completions/mean_terminated_length": 775.3212280273438, + "completions/min_length": 16.0, + "completions/min_terminated_length": 16.0, + "epoch": 0.3482119996586157, + "grad_norm": 1.1834700107574463, + "kl": 6.171875, + "learning_rate": 8.415069526291486e-07, + "loss": 0.3246, + "num_tokens": 604423158.0, + "reward": 1.67724609375, + "reward_std": 0.7229315042495728, + "rewards/accuracy_reward/mean": 0.123046875, + "rewards/accuracy_reward/std": 0.32881227135658264, + "rewards/format_reward/mean": 0.673828125, + "rewards/format_reward/std": 0.4692695140838623, + "rewards/tag_count_reward/mean": 0.88037109375, + "rewards/tag_count_reward/std": 0.21551933884620667, + "step": 1020 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2033.0, + "completions/mean_length": 875.41015625, + "completions/mean_terminated_length": 847.26806640625, + "completions/min_length": 34.0, + "completions/min_terminated_length": 34.0, + "epoch": 0.34855338397200647, + "grad_norm": 1.2013593912124634, + "kl": 5.6328125, + "learning_rate": 8.410983301215415e-07, + "loss": 0.3053, + "num_tokens": 604940456.0, + "reward": 1.69384765625, + "reward_std": 0.6755542159080505, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.71484375, + "rewards/format_reward/std": 0.45193037390708923, + "rewards/tag_count_reward/mean": 0.88720703125, + "rewards/tag_count_reward/std": 0.21295472979545593, + "step": 1021 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 855.462890625, + "completions/mean_terminated_length": 834.1251831054688, + "completions/min_length": 172.0, + "completions/min_terminated_length": 172.0, + "epoch": 0.3488947682853973, + "grad_norm": 3.499455213546753, + "kl": 6.1875, + "learning_rate": 8.406892944530973e-07, + "loss": 0.277, + "num_tokens": 605451221.0, + "reward": 1.6103515625, + "reward_std": 0.6938778758049011, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.673828125, + "rewards/format_reward/std": 0.4692695140838623, + "rewards/tag_count_reward/mean": 0.8720703125, + "rewards/tag_count_reward/std": 0.21271060407161713, + "step": 1022 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2005.0, + "completions/mean_length": 828.400390625, + "completions/mean_terminated_length": 801.6227416992188, + "completions/min_length": 125.0, + "completions/min_terminated_length": 125.0, + "epoch": 0.3492361525987881, + "grad_norm": 3.2929532527923584, + "kl": 6.3671875, + "learning_rate": 8.402798462043673e-07, + "loss": 0.2915, + "num_tokens": 605950674.0, + "reward": 1.61669921875, + "reward_std": 0.710044264793396, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.650390625, + "rewards/format_reward/std": 0.47731292247772217, + "rewards/tag_count_reward/mean": 0.87255859375, + "rewards/tag_count_reward/std": 0.2211734801530838, + "step": 1023 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 858.580078125, + "completions/mean_terminated_length": 825.1425170898438, + "completions/min_length": 9.0, + "completions/min_terminated_length": 9.0, + "epoch": 0.3495775369121789, + "grad_norm": 3.8285183906555176, + "kl": 6.796875, + "learning_rate": 8.398699859564887e-07, + "loss": 0.3447, + "num_tokens": 606455771.0, + "reward": 1.66162109375, + "reward_std": 0.7346779108047485, + "rewards/accuracy_reward/mean": 0.1270161271095276, + "rewards/accuracy_reward/std": 0.33332720398902893, + "rewards/format_reward/mean": 0.6640625, + "rewards/format_reward/std": 0.4727790653705597, + "rewards/tag_count_reward/mean": 0.87451171875, + "rewards/tag_count_reward/std": 0.2189633846282959, + "step": 1024 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1959.0, + "completions/mean_length": 893.4609375, + "completions/mean_terminated_length": 839.157470703125, + "completions/min_length": 50.0, + "completions/min_terminated_length": 50.0, + "epoch": 0.34991892122556967, + "grad_norm": 1.2652173042297363, + "kl": 5.65625, + "learning_rate": 8.39459714291183e-07, + "loss": 0.283, + "num_tokens": 606993927.0, + "reward": 1.5908203125, + "reward_std": 0.7199634909629822, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.640625, + "rewards/format_reward/std": 0.48028653860092163, + "rewards/tag_count_reward/mean": 0.8681640625, + "rewards/tag_count_reward/std": 0.22546352446079254, + "step": 1025 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 881.57421875, + "completions/mean_terminated_length": 819.1727905273438, + "completions/min_length": 76.0, + "completions/min_terminated_length": 76.0, + "epoch": 0.3502603055389605, + "grad_norm": 2.198275327682495, + "kl": 5.546875, + "learning_rate": 8.390490317907557e-07, + "loss": 0.3021, + "num_tokens": 607520765.0, + "reward": 1.5654296875, + "reward_std": 0.6948752403259277, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.62890625, + "rewards/format_reward/std": 0.4835699498653412, + "rewards/tag_count_reward/mean": 0.8779296875, + "rewards/tag_count_reward/std": 0.21271060407161713, + "step": 1026 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2024.0, + "completions/mean_length": 859.9921875, + "completions/mean_terminated_length": 819.1919555664062, + "completions/min_length": 214.0, + "completions/min_terminated_length": 214.0, + "epoch": 0.3506016898523513, + "grad_norm": 3.3149538040161133, + "kl": 4.89453125, + "learning_rate": 8.386379390380956e-07, + "loss": 0.2831, + "num_tokens": 608032809.0, + "reward": 1.68896484375, + "reward_std": 0.6970325708389282, + "rewards/accuracy_reward/mean": 0.1171875, + "rewards/accuracy_reward/std": 0.32195815443992615, + "rewards/format_reward/mean": 0.6875, + "rewards/format_reward/std": 0.4639657139778137, + "rewards/tag_count_reward/mean": 0.88427734375, + "rewards/tag_count_reward/std": 0.21424756944179535, + "step": 1027 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1970.0, + "completions/mean_length": 913.5859375, + "completions/mean_terminated_length": 860.22900390625, + "completions/min_length": 200.0, + "completions/min_terminated_length": 200.0, + "epoch": 0.3509430741657421, + "grad_norm": 3.4048895835876465, + "kl": 4.83984375, + "learning_rate": 8.382264366166736e-07, + "loss": 0.2893, + "num_tokens": 608572741.0, + "reward": 1.69482421875, + "reward_std": 0.7132976055145264, + "rewards/accuracy_reward/mean": 0.1328125, + "rewards/accuracy_reward/std": 0.33970388770103455, + "rewards/format_reward/mean": 0.6796875, + "rewards/format_reward/std": 0.4670529365539551, + "rewards/tag_count_reward/mean": 0.88232421875, + "rewards/tag_count_reward/std": 0.21772050857543945, + "step": 1028 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 914.361328125, + "completions/mean_terminated_length": 856.1663818359375, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "epoch": 0.35128445847913287, + "grad_norm": 1.075799584388733, + "kl": 5.4921875, + "learning_rate": 8.378145251105423e-07, + "loss": 0.2675, + "num_tokens": 609114430.0, + "reward": 1.484375, + "reward_std": 0.7132372856140137, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.59375, + "rewards/format_reward/std": 0.49161264300346375, + "rewards/tag_count_reward/mean": 0.84765625, + "rewards/tag_count_reward/std": 0.23716437816619873, + "step": 1029 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1959.0, + "completions/mean_length": 889.15625, + "completions/mean_terminated_length": 811.9000244140625, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "epoch": 0.3516258427925237, + "grad_norm": 1.336026668548584, + "kl": 6.1953125, + "learning_rate": 8.374022051043344e-07, + "loss": 0.3564, + "num_tokens": 609654062.0, + "reward": 1.5947265625, + "reward_std": 0.7138975262641907, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.638671875, + "rewards/format_reward/std": 0.48085519671440125, + "rewards/tag_count_reward/mean": 0.8623046875, + "rewards/tag_count_reward/std": 0.2358209192752838, + "step": 1030 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1946.0, + "completions/mean_length": 787.4921875, + "completions/mean_terminated_length": 762.3825073242188, + "completions/min_length": 113.0, + "completions/min_terminated_length": 113.0, + "epoch": 0.3519672271059145, + "grad_norm": 2.8935294151306152, + "kl": 4.9140625, + "learning_rate": 8.36989477183263e-07, + "loss": 0.2718, + "num_tokens": 610135434.0, + "reward": 1.62109375, + "reward_std": 0.6810424327850342, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.646484375, + "rewards/format_reward/std": 0.47852855920791626, + "rewards/tag_count_reward/mean": 0.89453125, + "rewards/tag_count_reward/std": 0.20177629590034485, + "step": 1031 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1990.0, + "completions/mean_length": 808.677734375, + "completions/mean_terminated_length": 768.6995849609375, + "completions/min_length": 4.0, + "completions/min_terminated_length": 4.0, + "epoch": 0.3523086114193053, + "grad_norm": 1.5922437906265259, + "kl": 4.234375, + "learning_rate": 8.365763419331199e-07, + "loss": 0.1738, + "num_tokens": 610625189.0, + "reward": 1.63134765625, + "reward_std": 0.6297500729560852, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.6953125, + "rewards/format_reward/std": 0.4607250988483429, + "rewards/tag_count_reward/mean": 0.89111328125, + "rewards/tag_count_reward/std": 0.2038862407207489, + "step": 1032 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1990.0, + "completions/mean_length": 843.97265625, + "completions/mean_terminated_length": 822.4293823242188, + "completions/min_length": 160.0, + "completions/min_terminated_length": 160.0, + "epoch": 0.35264999573269606, + "grad_norm": 1.8940683603286743, + "kl": 4.8515625, + "learning_rate": 8.361627999402748e-07, + "loss": 0.1957, + "num_tokens": 611133239.0, + "reward": 1.61669921875, + "reward_std": 0.6727503538131714, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.669921875, + "rewards/format_reward/std": 0.47070086002349854, + "rewards/tag_count_reward/mean": 0.87841796875, + "rewards/tag_count_reward/std": 0.21612590551376343, + "step": 1033 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1943.0, + "completions/mean_length": 789.708984375, + "completions/mean_terminated_length": 762.0818481445312, + "completions/min_length": 35.0, + "completions/min_terminated_length": 35.0, + "epoch": 0.3529913800460869, + "grad_norm": 1.276130199432373, + "kl": 5.2109375, + "learning_rate": 8.357488517916752e-07, + "loss": 0.2572, + "num_tokens": 611609122.0, + "reward": 1.68603515625, + "reward_std": 0.6626811623573303, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.697265625, + "rewards/format_reward/std": 0.45989060401916504, + "rewards/tag_count_reward/mean": 0.89306640625, + "rewards/tag_count_reward/std": 0.21138295531272888, + "step": 1034 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2016.0, + "completions/mean_length": 888.078125, + "completions/mean_terminated_length": 840.9268188476562, + "completions/min_length": 122.0, + "completions/min_terminated_length": 122.0, + "epoch": 0.3533327643594777, + "grad_norm": 1.1660982370376587, + "kl": 5.3359375, + "learning_rate": 8.353344980748446e-07, + "loss": 0.2584, + "num_tokens": 612142346.0, + "reward": 1.60986328125, + "reward_std": 0.6833149194717407, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.658203125, + "rewards/format_reward/std": 0.4747757613658905, + "rewards/tag_count_reward/mean": 0.88330078125, + "rewards/tag_count_reward/std": 0.210835263133049, + "step": 1035 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 841.369140625, + "completions/mean_terminated_length": 812.4100341796875, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "epoch": 0.3536741486728685, + "grad_norm": 1.5263030529022217, + "kl": 3.9296875, + "learning_rate": 8.349197393778825e-07, + "loss": 0.1906, + "num_tokens": 612654679.0, + "reward": 1.68212890625, + "reward_std": 0.6412637233734131, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.705078125, + "rewards/format_reward/std": 0.4564536213874817, + "rewards/tag_count_reward/mean": 0.90087890625, + "rewards/tag_count_reward/std": 0.20467674732208252, + "step": 1036 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2029.0, + "completions/mean_length": 927.36328125, + "completions/mean_terminated_length": 862.5330200195312, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "epoch": 0.35401553298625926, + "grad_norm": 1.7529670000076294, + "kl": 4.578125, + "learning_rate": 8.345045762894628e-07, + "loss": 0.2572, + "num_tokens": 613204161.0, + "reward": 1.623046875, + "reward_std": 0.6420872211456299, + "rewards/accuracy_reward/mean": 0.025390625, + "rewards/accuracy_reward/std": 0.15746226906776428, + "rewards/format_reward/mean": 0.712890625, + "rewards/format_reward/std": 0.45285552740097046, + "rewards/tag_count_reward/mean": 0.884765625, + "rewards/tag_count_reward/std": 0.22041666507720947, + "step": 1037 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2041.0, + "completions/mean_length": 954.96875, + "completions/mean_terminated_length": 896.4937744140625, + "completions/min_length": 210.0, + "completions/min_terminated_length": 210.0, + "epoch": 0.3543569172996501, + "grad_norm": 1.533047080039978, + "kl": 4.6484375, + "learning_rate": 8.340890093988336e-07, + "loss": 0.2689, + "num_tokens": 613764225.0, + "reward": 1.6484375, + "reward_std": 0.6554567813873291, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.701171875, + "rewards/format_reward/std": 0.45819199085235596, + "rewards/tag_count_reward/mean": 0.888671875, + "rewards/tag_count_reward/std": 0.2117132544517517, + "step": 1038 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 851.8046875, + "completions/mean_terminated_length": 823.0960693359375, + "completions/min_length": 18.0, + "completions/min_terminated_length": 18.0, + "epoch": 0.3546983016130409, + "grad_norm": 2.632937431335449, + "kl": 3.77734375, + "learning_rate": 8.336730392958163e-07, + "loss": 0.2126, + "num_tokens": 614272989.0, + "reward": 1.77001953125, + "reward_std": 0.5945524573326111, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.76171875, + "rewards/format_reward/std": 0.42644867300987244, + "rewards/tag_count_reward/mean": 0.92236328125, + "rewards/tag_count_reward/std": 0.17819732427597046, + "step": 1039 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 842.3046875, + "completions/mean_terminated_length": 790.7373046875, + "completions/min_length": 8.0, + "completions/min_terminated_length": 8.0, + "epoch": 0.3550396859264317, + "grad_norm": 2.310908794403076, + "kl": 5.375, + "learning_rate": 8.332566665708041e-07, + "loss": 0.3472, + "num_tokens": 614776889.0, + "reward": 1.7568359375, + "reward_std": 0.694198489189148, + "rewards/accuracy_reward/mean": 0.119140625, + "rewards/accuracy_reward/std": 0.32427072525024414, + "rewards/format_reward/mean": 0.7421875, + "rewards/format_reward/std": 0.43785804510116577, + "rewards/tag_count_reward/mean": 0.8955078125, + "rewards/tag_count_reward/std": 0.21057961881160736, + "step": 1040 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 787.267578125, + "completions/mean_terminated_length": 751.8252563476562, + "completions/min_length": 159.0, + "completions/min_terminated_length": 159.0, + "epoch": 0.35538107023982246, + "grad_norm": 2.5846283435821533, + "kl": 5.9921875, + "learning_rate": 8.328398918147622e-07, + "loss": 0.3324, + "num_tokens": 615262434.0, + "reward": 1.67724609375, + "reward_std": 0.6175016760826111, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.7265625, + "rewards/format_reward/std": 0.4461594223976135, + "rewards/tag_count_reward/mean": 0.88623046875, + "rewards/tag_count_reward/std": 0.22145411372184753, + "step": 1041 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1964.0, + "completions/mean_length": 830.2890625, + "completions/mean_terminated_length": 773.0142822265625, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "epoch": 0.3557224545532133, + "grad_norm": 5.108445644378662, + "kl": 7.3046875, + "learning_rate": 8.32422715619226e-07, + "loss": 0.4022, + "num_tokens": 615761830.0, + "reward": 1.66845703125, + "reward_std": 0.6759384274482727, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.69921875, + "rewards/format_reward/std": 0.45904624462127686, + "rewards/tag_count_reward/mean": 0.87744140625, + "rewards/tag_count_reward/std": 0.22878453135490417, + "step": 1042 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1935.0, + "completions/mean_length": 924.916015625, + "completions/mean_terminated_length": 876.8818969726562, + "completions/min_length": 163.0, + "completions/min_terminated_length": 163.0, + "epoch": 0.3560638388666041, + "grad_norm": 2.7219574451446533, + "kl": 5.99609375, + "learning_rate": 8.320051385763005e-07, + "loss": 0.32, + "num_tokens": 616322107.0, + "reward": 1.66943359375, + "reward_std": 0.6089550256729126, + "rewards/accuracy_reward/mean": 0.058467742055654526, + "rewards/accuracy_reward/std": 0.23486268520355225, + "rewards/format_reward/mean": 0.7109375, + "rewards/format_reward/std": 0.45377036929130554, + "rewards/tag_count_reward/mean": 0.90185546875, + "rewards/tag_count_reward/std": 0.19348861277103424, + "step": 1043 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2001.0, + "completions/mean_length": 845.603515625, + "completions/mean_terminated_length": 794.17724609375, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "epoch": 0.3564052231799949, + "grad_norm": 2.1691174507141113, + "kl": 6.421875, + "learning_rate": 8.315871612786604e-07, + "loss": 0.345, + "num_tokens": 616840768.0, + "reward": 1.6875, + "reward_std": 0.692090630531311, + "rewards/accuracy_reward/mean": 0.11088709533214569, + "rewards/accuracy_reward/std": 0.3143092691898346, + "rewards/format_reward/mean": 0.697265625, + "rewards/format_reward/std": 0.45989060401916504, + "rewards/tag_count_reward/mean": 0.8828125, + "rewards/tag_count_reward/std": 0.22434400022029877, + "step": 1044 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.056640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 823.98828125, + "completions/mean_terminated_length": 750.4968872070312, + "completions/min_length": 163.0, + "completions/min_terminated_length": 163.0, + "epoch": 0.35674660749338566, + "grad_norm": 1.5825062990188599, + "kl": 6.8515625, + "learning_rate": 8.311687843195481e-07, + "loss": 0.4378, + "num_tokens": 617346394.0, + "reward": 1.66796875, + "reward_std": 0.6295381784439087, + "rewards/accuracy_reward/mean": 0.08064515888690948, + "rewards/accuracy_reward/std": 0.2725643217563629, + "rewards/format_reward/mean": 0.7109375, + "rewards/format_reward/std": 0.45377036929130554, + "rewards/tag_count_reward/mean": 0.87890625, + "rewards/tag_count_reward/std": 0.2308928668498993, + "step": 1045 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 857.66015625, + "completions/mean_terminated_length": 799.1188354492188, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "epoch": 0.3570879918067765, + "grad_norm": 2.7481093406677246, + "kl": 4.765625, + "learning_rate": 8.307500082927726e-07, + "loss": 0.2886, + "num_tokens": 617863468.0, + "reward": 1.62939453125, + "reward_std": 0.6399465799331665, + "rewards/accuracy_reward/mean": 0.03125, + "rewards/accuracy_reward/std": 0.17416280508041382, + "rewards/format_reward/mean": 0.708984375, + "rewards/format_reward/std": 0.45467492938041687, + "rewards/tag_count_reward/mean": 0.88916015625, + "rewards/tag_count_reward/std": 0.22238846123218536, + "step": 1046 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1773.0, + "completions/mean_length": 808.796875, + "completions/mean_terminated_length": 763.6437377929688, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "epoch": 0.3574293761201673, + "grad_norm": 2.5364630222320557, + "kl": 4.58984375, + "learning_rate": 8.303308337927103e-07, + "loss": 0.2764, + "num_tokens": 618359188.0, + "reward": 1.73046875, + "reward_std": 0.656836211681366, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.74609375, + "rewards/format_reward/std": 0.43567025661468506, + "rewards/tag_count_reward/mean": 0.90234375, + "rewards/tag_count_reward/std": 0.20686452090740204, + "step": 1047 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 781.765625, + "completions/mean_terminated_length": 727.6090087890625, + "completions/min_length": 40.0, + "completions/min_terminated_length": 40.0, + "epoch": 0.3577707604335581, + "grad_norm": 4.034943580627441, + "kl": 5.46875, + "learning_rate": 8.299112614143028e-07, + "loss": 0.3919, + "num_tokens": 618843836.0, + "reward": 1.7705078125, + "reward_std": 0.6315006017684937, + "rewards/accuracy_reward/mean": 0.115234375, + "rewards/accuracy_reward/std": 0.3196168541908264, + "rewards/format_reward/mean": 0.75, + "rewards/format_reward/std": 0.43343618512153625, + "rewards/tag_count_reward/mean": 0.9052734375, + "rewards/tag_count_reward/std": 0.1986045390367508, + "step": 1048 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 826.744140625, + "completions/mean_terminated_length": 771.9122314453125, + "completions/min_length": 38.0, + "completions/min_terminated_length": 38.0, + "epoch": 0.35811214474694886, + "grad_norm": 3.5188894271850586, + "kl": 6.734375, + "learning_rate": 8.29491291753056e-07, + "loss": 0.3553, + "num_tokens": 619348121.0, + "reward": 1.6513671875, + "reward_std": 0.7160419225692749, + "rewards/accuracy_reward/mean": 0.1015625, + "rewards/accuracy_reward/std": 0.30236753821372986, + "rewards/format_reward/mean": 0.669921875, + "rewards/format_reward/std": 0.47070086002349854, + "rewards/tag_count_reward/mean": 0.8798828125, + "rewards/tag_count_reward/std": 0.22497136890888214, + "step": 1049 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1984.0, + "completions/mean_length": 863.94921875, + "completions/mean_terminated_length": 815.8170166015625, + "completions/min_length": 145.0, + "completions/min_terminated_length": 145.0, + "epoch": 0.35845352906033967, + "grad_norm": 3.561249256134033, + "kl": 6.609375, + "learning_rate": 8.290709254050403e-07, + "loss": 0.3654, + "num_tokens": 619872239.0, + "reward": 1.58251953125, + "reward_std": 0.6410717368125916, + "rewards/accuracy_reward/mean": 0.030241934582591057, + "rewards/accuracy_reward/std": 0.1714252382516861, + "rewards/format_reward/mean": 0.677734375, + "rewards/format_reward/std": 0.46780112385749817, + "rewards/tag_count_reward/mean": 0.87548828125, + "rewards/tag_count_reward/std": 0.22447973489761353, + "step": 1050 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 785.029296875, + "completions/mean_terminated_length": 739.0101318359375, + "completions/min_length": 32.0, + "completions/min_terminated_length": 32.0, + "epoch": 0.3587949133737305, + "grad_norm": 2.92173171043396, + "kl": 6.84375, + "learning_rate": 8.286501629668887e-07, + "loss": 0.3545, + "num_tokens": 620348702.0, + "reward": 1.66162109375, + "reward_std": 0.6889419555664062, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.689453125, + "rewards/format_reward/std": 0.46317005157470703, + "rewards/tag_count_reward/mean": 0.87646484375, + "rewards/tag_count_reward/std": 0.22556257247924805, + "step": 1051 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2022.0, + "completions/mean_length": 833.7734375, + "completions/mean_terminated_length": 789.5303955078125, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "epoch": 0.3591362976871213, + "grad_norm": 1.6500178575515747, + "kl": 6.25, + "learning_rate": 8.282290050357966e-07, + "loss": 0.3758, + "num_tokens": 620848810.0, + "reward": 1.61376953125, + "reward_std": 0.6589176058769226, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.677734375, + "rewards/format_reward/std": 0.46780112385749817, + "rewards/tag_count_reward/mean": 0.88720703125, + "rewards/tag_count_reward/std": 0.21410034596920013, + "step": 1052 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2031.0, + "completions/mean_length": 862.044921875, + "completions/mean_terminated_length": 816.3387451171875, + "completions/min_length": 9.0, + "completions/min_terminated_length": 9.0, + "epoch": 0.35947768200051206, + "grad_norm": 3.1046407222747803, + "kl": 5.515625, + "learning_rate": 8.278074522095207e-07, + "loss": 0.3305, + "num_tokens": 621370817.0, + "reward": 1.630859375, + "reward_std": 0.6795388460159302, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.69140625, + "rewards/format_reward/std": 0.4623647928237915, + "rewards/tag_count_reward/mean": 0.8828125, + "rewards/tag_count_reward/std": 0.22160130739212036, + "step": 1053 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2018.0, + "completions/mean_length": 877.873046875, + "completions/mean_terminated_length": 827.826904296875, + "completions/min_length": 108.0, + "completions/min_terminated_length": 108.0, + "epoch": 0.35981906631390287, + "grad_norm": 2.707383155822754, + "kl": 4.17578125, + "learning_rate": 8.273855050863779e-07, + "loss": 0.2529, + "num_tokens": 621906192.0, + "reward": 1.70751953125, + "reward_std": 0.6367489099502563, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.73828125, + "rewards/format_reward/std": 0.44000017642974854, + "rewards/tag_count_reward/mean": 0.90087890625, + "rewards/tag_count_reward/std": 0.20467674732208252, + "step": 1054 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1987.0, + "completions/mean_length": 807.484375, + "completions/mean_terminated_length": 764.880859375, + "completions/min_length": 126.0, + "completions/min_terminated_length": 126.0, + "epoch": 0.3601604506272937, + "grad_norm": 1.5204405784606934, + "kl": 5.0625, + "learning_rate": 8.269631642652454e-07, + "loss": 0.2856, + "num_tokens": 622394088.0, + "reward": 1.72119140625, + "reward_std": 0.6696908473968506, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.728515625, + "rewards/format_reward/std": 0.44516023993492126, + "rewards/tag_count_reward/mean": 0.89892578125, + "rewards/tag_count_reward/std": 0.21195626258850098, + "step": 1055 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1985.0, + "completions/mean_length": 826.232421875, + "completions/mean_terminated_length": 779.14599609375, + "completions/min_length": 157.0, + "completions/min_terminated_length": 157.0, + "epoch": 0.3605018349406845, + "grad_norm": 0.9496316909790039, + "kl": 5.234375, + "learning_rate": 8.265404303455583e-07, + "loss": 0.2785, + "num_tokens": 622892383.0, + "reward": 1.69775390625, + "reward_std": 0.6280603408813477, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.744140625, + "rewards/format_reward/std": 0.43676990270614624, + "rewards/tag_count_reward/mean": 0.90087890625, + "rewards/tag_count_reward/std": 0.20287610590457916, + "step": 1056 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2025.0, + "completions/mean_length": 814.70703125, + "completions/mean_terminated_length": 772.3515625, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "epoch": 0.36084321925407525, + "grad_norm": 4.240236282348633, + "kl": 6.4453125, + "learning_rate": 8.261173039273103e-07, + "loss": 0.3098, + "num_tokens": 623380057.0, + "reward": 1.67333984375, + "reward_std": 0.6242455244064331, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.720703125, + "rewards/format_reward/std": 0.44909247756004333, + "rewards/tag_count_reward/mean": 0.89794921875, + "rewards/tag_count_reward/std": 0.20382998883724213, + "step": 1057 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2013.0, + "completions/mean_length": 845.03515625, + "completions/mean_terminated_length": 803.7212524414062, + "completions/min_length": 130.0, + "completions/min_terminated_length": 130.0, + "epoch": 0.36118460356746607, + "grad_norm": 2.6613004207611084, + "kl": 6.1015625, + "learning_rate": 8.25693785611052e-07, + "loss": 0.3315, + "num_tokens": 623892971.0, + "reward": 1.63720703125, + "reward_std": 0.6636764407157898, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.69921875, + "rewards/format_reward/std": 0.45904624462127686, + "rewards/tag_count_reward/mean": 0.87548828125, + "rewards/tag_count_reward/std": 0.21952125430107117, + "step": 1058 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 856.583984375, + "completions/mean_terminated_length": 820.625732421875, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "epoch": 0.3615259878808569, + "grad_norm": 2.4534904956817627, + "kl": 6.5234375, + "learning_rate": 8.2526987599789e-07, + "loss": 0.3675, + "num_tokens": 624397830.0, + "reward": 1.689453125, + "reward_std": 0.6934724450111389, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.716796875, + "rewards/format_reward/std": 0.4509948492050171, + "rewards/tag_count_reward/mean": 0.888671875, + "rewards/tag_count_reward/std": 0.21909481287002563, + "step": 1059 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1960.0, + "completions/mean_length": 784.845703125, + "completions/mean_terminated_length": 738.8198852539062, + "completions/min_length": 76.0, + "completions/min_terminated_length": 76.0, + "epoch": 0.3618673721942477, + "grad_norm": 1.810727596282959, + "kl": 4.63671875, + "learning_rate": 8.248455756894865e-07, + "loss": 0.2552, + "num_tokens": 624874919.0, + "reward": 1.7666015625, + "reward_std": 0.6343519687652588, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.767578125, + "rewards/format_reward/std": 0.42278963327407837, + "rewards/tag_count_reward/mean": 0.9072265625, + "rewards/tag_count_reward/std": 0.20437131822109222, + "step": 1060 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2010.0, + "completions/mean_length": 855.83203125, + "completions/mean_terminated_length": 812.3927612304688, + "completions/min_length": 198.0, + "completions/min_terminated_length": 198.0, + "epoch": 0.36220875650763845, + "grad_norm": 1.5865952968597412, + "kl": 4.54296875, + "learning_rate": 8.244208852880583e-07, + "loss": 0.2585, + "num_tokens": 625403969.0, + "reward": 1.7119140625, + "reward_std": 0.6156963109970093, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.720703125, + "rewards/format_reward/std": 0.44909247756004333, + "rewards/tag_count_reward/mean": 0.8935546875, + "rewards/tag_count_reward/std": 0.203678160905838, + "step": 1061 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 742.169921875, + "completions/mean_terminated_length": 716.1574096679688, + "completions/min_length": 133.0, + "completions/min_terminated_length": 133.0, + "epoch": 0.36255014082102927, + "grad_norm": 1.674841284751892, + "kl": 4.56640625, + "learning_rate": 8.239958053963758e-07, + "loss": 0.2619, + "num_tokens": 625857752.0, + "reward": 1.765625, + "reward_std": 0.5894856452941895, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.7578125, + "rewards/format_reward/std": 0.42882615327835083, + "rewards/tag_count_reward/mean": 0.912109375, + "rewards/tag_count_reward/std": 0.18196536600589752, + "step": 1062 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2019.0, + "completions/mean_length": 682.232421875, + "completions/mean_terminated_length": 666.03759765625, + "completions/min_length": 145.0, + "completions/min_terminated_length": 145.0, + "epoch": 0.3628915251344201, + "grad_norm": 2.8770751953125, + "kl": 4.38671875, + "learning_rate": 8.23570336617762e-07, + "loss": 0.2604, + "num_tokens": 626275391.0, + "reward": 1.80908203125, + "reward_std": 0.6140027642250061, + "rewards/accuracy_reward/mean": 0.11328125, + "rewards/accuracy_reward/std": 0.3172462284564972, + "rewards/format_reward/mean": 0.7734375, + "rewards/format_reward/std": 0.4190165400505066, + "rewards/tag_count_reward/mean": 0.92236328125, + "rewards/tag_count_reward/std": 0.1775096207857132, + "step": 1063 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 821.57421875, + "completions/mean_terminated_length": 779.45458984375, + "completions/min_length": 158.0, + "completions/min_terminated_length": 158.0, + "epoch": 0.3632329094478109, + "grad_norm": 2.4590604305267334, + "kl": 6.3984375, + "learning_rate": 8.23144479556092e-07, + "loss": 0.3595, + "num_tokens": 626775701.0, + "reward": 1.66259765625, + "reward_std": 0.694244921207428, + "rewards/accuracy_reward/mean": 0.087890625, + "rewards/accuracy_reward/std": 0.2834126651287079, + "rewards/format_reward/mean": 0.6953125, + "rewards/format_reward/std": 0.4607250988483429, + "rewards/tag_count_reward/mean": 0.87939453125, + "rewards/tag_count_reward/std": 0.21947772800922394, + "step": 1064 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1975.0, + "completions/mean_length": 817.7109375, + "completions/mean_terminated_length": 778.024169921875, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "epoch": 0.36357429376120165, + "grad_norm": 1.5840237140655518, + "kl": 5.2578125, + "learning_rate": 8.227182348157923e-07, + "loss": 0.2916, + "num_tokens": 627268497.0, + "reward": 1.716796875, + "reward_std": 0.6501511335372925, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.748046875, + "rewards/format_reward/std": 0.43455907702445984, + "rewards/tag_count_reward/mean": 0.89453125, + "rewards/tag_count_reward/std": 0.20774950087070465, + "step": 1065 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2031.0, + "completions/mean_length": 790.078125, + "completions/mean_terminated_length": 765.0199584960938, + "completions/min_length": 196.0, + "completions/min_terminated_length": 196.0, + "epoch": 0.36391567807459246, + "grad_norm": 2.5225985050201416, + "kl": 5.8828125, + "learning_rate": 8.222916030018389e-07, + "loss": 0.3334, + "num_tokens": 627744697.0, + "reward": 1.7060546875, + "reward_std": 0.6291856169700623, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.724609375, + "rewards/format_reward/std": 0.44714778661727905, + "rewards/tag_count_reward/mean": 0.8876953125, + "rewards/tag_count_reward/std": 0.2082800716161728, + "step": 1066 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1969.0, + "completions/mean_length": 772.548828125, + "completions/mean_terminated_length": 739.3206787109375, + "completions/min_length": 36.0, + "completions/min_terminated_length": 36.0, + "epoch": 0.3642570623879833, + "grad_norm": 1.46543288230896, + "kl": 4.73828125, + "learning_rate": 8.21864584719758e-07, + "loss": 0.254, + "num_tokens": 628220450.0, + "reward": 1.75927734375, + "reward_std": 0.6168092489242554, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.779296875, + "rewards/format_reward/std": 0.4151262938976288, + "rewards/tag_count_reward/mean": 0.91357421875, + "rewards/tag_count_reward/std": 0.19466042518615723, + "step": 1067 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1986.0, + "completions/mean_length": 822.328125, + "completions/mean_terminated_length": 785.3359985351562, + "completions/min_length": 135.0, + "completions/min_terminated_length": 135.0, + "epoch": 0.3645984467013741, + "grad_norm": 1.9685052633285522, + "kl": 4.9921875, + "learning_rate": 8.214371805756238e-07, + "loss": 0.2502, + "num_tokens": 628718378.0, + "reward": 1.7216796875, + "reward_std": 0.6598231196403503, + "rewards/accuracy_reward/mean": 0.10282257944345474, + "rewards/accuracy_reward/std": 0.30403366684913635, + "rewards/format_reward/mean": 0.732421875, + "rewards/format_reward/std": 0.4431293308734894, + "rewards/tag_count_reward/mean": 0.8896484375, + "rewards/tag_count_reward/std": 0.21451778709888458, + "step": 1068 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2012.0, + "completions/mean_length": 888.525390625, + "completions/mean_terminated_length": 836.46728515625, + "completions/min_length": 135.0, + "completions/min_terminated_length": 135.0, + "epoch": 0.36493983101476485, + "grad_norm": 2.010416269302368, + "kl": 4.625, + "learning_rate": 8.210093911760582e-07, + "loss": 0.2752, + "num_tokens": 629254487.0, + "reward": 1.66357421875, + "reward_std": 0.6304594874382019, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.720703125, + "rewards/format_reward/std": 0.44909247756004333, + "rewards/tag_count_reward/mean": 0.89404296875, + "rewards/tag_count_reward/std": 0.20121611654758453, + "step": 1069 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 770.421875, + "completions/mean_terminated_length": 744.9721069335938, + "completions/min_length": 56.0, + "completions/min_terminated_length": 56.0, + "epoch": 0.36528121532815566, + "grad_norm": 1.9328861236572266, + "kl": 4.06640625, + "learning_rate": 8.205812171282302e-07, + "loss": 0.2321, + "num_tokens": 629727439.0, + "reward": 1.7705078125, + "reward_std": 0.5791828036308289, + "rewards/accuracy_reward/mean": 0.07459677755832672, + "rewards/accuracy_reward/std": 0.263004869222641, + "rewards/format_reward/mean": 0.78125, + "rewards/format_reward/std": 0.41380295157432556, + "rewards/tag_count_reward/mean": 0.9169921875, + "rewards/tag_count_reward/std": 0.18883807957172394, + "step": 1070 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2031.0, + "completions/mean_length": 850.46875, + "completions/mean_terminated_length": 809.3414306640625, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "epoch": 0.3656225996415465, + "grad_norm": 1.2636034488677979, + "kl": 4.890625, + "learning_rate": 8.201526590398543e-07, + "loss": 0.2682, + "num_tokens": 630247391.0, + "reward": 1.70361328125, + "reward_std": 0.6831998825073242, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.724609375, + "rewards/format_reward/std": 0.44714778661727905, + "rewards/tag_count_reward/mean": 0.88525390625, + "rewards/tag_count_reward/std": 0.22039444744586945, + "step": 1071 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 868.84765625, + "completions/mean_terminated_length": 820.9146118164062, + "completions/min_length": 28.0, + "completions/min_terminated_length": 28.0, + "epoch": 0.3659639839549373, + "grad_norm": 1.3227003812789917, + "kl": 5.53125, + "learning_rate": 8.197237175191907e-07, + "loss": 0.2956, + "num_tokens": 630769969.0, + "reward": 1.7021484375, + "reward_std": 0.633798360824585, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.732421875, + "rewards/format_reward/std": 0.4431293308734894, + "rewards/tag_count_reward/mean": 0.8974609375, + "rewards/tag_count_reward/std": 0.2121168076992035, + "step": 1072 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1994.0, + "completions/mean_length": 778.013671875, + "completions/mean_terminated_length": 768.0137939453125, + "completions/min_length": 32.0, + "completions/min_terminated_length": 32.0, + "epoch": 0.36630536826832805, + "grad_norm": 1.466387152671814, + "kl": 4.18359375, + "learning_rate": 8.192943931750431e-07, + "loss": 0.2304, + "num_tokens": 631249336.0, + "reward": 1.73974609375, + "reward_std": 0.5979659557342529, + "rewards/accuracy_reward/mean": 0.08669354766607285, + "rewards/accuracy_reward/std": 0.281669557094574, + "rewards/format_reward/mean": 0.74609375, + "rewards/format_reward/std": 0.43567025661468506, + "rewards/tag_count_reward/mean": 0.90966796875, + "rewards/tag_count_reward/std": 0.19413939118385315, + "step": 1073 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1942.0, + "completions/mean_length": 793.66015625, + "completions/mean_terminated_length": 768.67333984375, + "completions/min_length": 170.0, + "completions/min_terminated_length": 170.0, + "epoch": 0.36664675258171886, + "grad_norm": 1.0847512483596802, + "kl": 5.2890625, + "learning_rate": 8.188646866167591e-07, + "loss": 0.2889, + "num_tokens": 631726858.0, + "reward": 1.75, + "reward_std": 0.6973233819007874, + "rewards/accuracy_reward/mean": 0.11290322244167328, + "rewards/accuracy_reward/std": 0.3167939782142639, + "rewards/format_reward/mean": 0.748046875, + "rewards/format_reward/std": 0.43455907702445984, + "rewards/tag_count_reward/mean": 0.892578125, + "rewards/tag_count_reward/std": 0.21486715972423553, + "step": 1074 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1970.0, + "completions/mean_length": 803.58203125, + "completions/mean_terminated_length": 778.7928466796875, + "completions/min_length": 41.0, + "completions/min_terminated_length": 41.0, + "epoch": 0.3669881368951097, + "grad_norm": 1.3949421644210815, + "kl": 4.5703125, + "learning_rate": 8.184345984542283e-07, + "loss": 0.2707, + "num_tokens": 632215252.0, + "reward": 1.76611328125, + "reward_std": 0.6225503087043762, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.759765625, + "rewards/format_reward/std": 0.4276435375213623, + "rewards/tag_count_reward/mean": 0.91064453125, + "rewards/tag_count_reward/std": 0.19206106662750244, + "step": 1075 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1977.0, + "completions/mean_length": 756.611328125, + "completions/mean_terminated_length": 733.5049438476562, + "completions/min_length": 9.0, + "completions/min_terminated_length": 9.0, + "epoch": 0.3673295212085005, + "grad_norm": 1.2364869117736816, + "kl": 5.0, + "learning_rate": 8.180041292978826e-07, + "loss": 0.2846, + "num_tokens": 632681005.0, + "reward": 1.79443359375, + "reward_std": 0.6380493640899658, + "rewards/accuracy_reward/mean": 0.125, + "rewards/accuracy_reward/std": 0.3310423493385315, + "rewards/format_reward/mean": 0.76171875, + "rewards/format_reward/std": 0.42644867300987244, + "rewards/tag_count_reward/mean": 0.90771484375, + "rewards/tag_count_reward/std": 0.19944652915000916, + "step": 1076 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 790.45703125, + "completions/mean_terminated_length": 767.9562377929688, + "completions/min_length": 8.0, + "completions/min_terminated_length": 8.0, + "epoch": 0.36767090552189124, + "grad_norm": 1.0173770189285278, + "kl": 5.1875, + "learning_rate": 8.175732797586939e-07, + "loss": 0.2963, + "num_tokens": 633153223.0, + "reward": 1.712890625, + "reward_std": 0.6708530783653259, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.74609375, + "rewards/format_reward/std": 0.43567025661468506, + "rewards/tag_count_reward/mean": 0.892578125, + "rewards/tag_count_reward/std": 0.21656812727451324, + "step": 1077 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2016.0, + "completions/mean_length": 821.943359375, + "completions/mean_terminated_length": 795.02392578125, + "completions/min_length": 166.0, + "completions/min_terminated_length": 166.0, + "epoch": 0.36801228983528206, + "grad_norm": 1.6971614360809326, + "kl": 5.8671875, + "learning_rate": 8.171420504481743e-07, + "loss": 0.3324, + "num_tokens": 633641386.0, + "reward": 1.6669921875, + "reward_std": 0.6330738067626953, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.7265625, + "rewards/format_reward/std": 0.4461594223976135, + "rewards/tag_count_reward/mean": 0.8994140625, + "rewards/tag_count_reward/std": 0.19879689812660217, + "step": 1078 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2000.0, + "completions/mean_length": 785.74609375, + "completions/mean_terminated_length": 729.0734252929688, + "completions/min_length": 181.0, + "completions/min_terminated_length": 181.0, + "epoch": 0.36835367414867287, + "grad_norm": 1.5560015439987183, + "kl": 6.8671875, + "learning_rate": 8.167104419783753e-07, + "loss": 0.4281, + "num_tokens": 634127256.0, + "reward": 1.7783203125, + "reward_std": 0.678869366645813, + "rewards/accuracy_reward/mean": 0.123046875, + "rewards/accuracy_reward/std": 0.32881227135658264, + "rewards/format_reward/mean": 0.7578125, + "rewards/format_reward/std": 0.42882615327835083, + "rewards/tag_count_reward/mean": 0.8974609375, + "rewards/tag_count_reward/std": 0.22225362062454224, + "step": 1079 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2008.0, + "completions/mean_length": 809.8359375, + "completions/mean_terminated_length": 787.681884765625, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "epoch": 0.3686950584620637, + "grad_norm": 2.276832342147827, + "kl": 5.34375, + "learning_rate": 8.162784549618855e-07, + "loss": 0.2815, + "num_tokens": 634623476.0, + "reward": 1.6962890625, + "reward_std": 0.649118185043335, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.728515625, + "rewards/format_reward/std": 0.44516023993492126, + "rewards/tag_count_reward/mean": 0.8974609375, + "rewards/tag_count_reward/std": 0.2050807625055313, + "step": 1080 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1952.0, + "completions/mean_length": 822.41796875, + "completions/mean_terminated_length": 787.9638061523438, + "completions/min_length": 222.0, + "completions/min_terminated_length": 222.0, + "epoch": 0.36903644277545444, + "grad_norm": 2.9448065757751465, + "kl": 4.58984375, + "learning_rate": 8.158460900118321e-07, + "loss": 0.3172, + "num_tokens": 635122042.0, + "reward": 1.77880859375, + "reward_std": 0.5868085622787476, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.7890625, + "rewards/format_reward/std": 0.4083731174468994, + "rewards/tag_count_reward/mean": 0.92138671875, + "rewards/tag_count_reward/std": 0.1838558167219162, + "step": 1081 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2041.0, + "completions/mean_length": 879.923828125, + "completions/mean_terminated_length": 817.4341430664062, + "completions/min_length": 185.0, + "completions/min_terminated_length": 185.0, + "epoch": 0.36937782708884526, + "grad_norm": 1.9693816900253296, + "kl": 5.25, + "learning_rate": 8.15413347741878e-07, + "loss": 0.3493, + "num_tokens": 635643347.0, + "reward": 1.67431640625, + "reward_std": 0.6367882490158081, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.736328125, + "rewards/format_reward/std": 0.4410543739795685, + "rewards/tag_count_reward/mean": 0.88916015625, + "rewards/tag_count_reward/std": 0.220177561044693, + "step": 1082 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2030.0, + "completions/mean_length": 871.9296875, + "completions/mean_terminated_length": 824.1219482421875, + "completions/min_length": 207.0, + "completions/min_terminated_length": 207.0, + "epoch": 0.36971921140223607, + "grad_norm": 1.6949392557144165, + "kl": 5.15625, + "learning_rate": 8.149802287662214e-07, + "loss": 0.3026, + "num_tokens": 636168431.0, + "reward": 1.6943359375, + "reward_std": 0.6163403987884521, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.71875, + "rewards/format_reward/std": 0.45004892349243164, + "rewards/tag_count_reward/mean": 0.8955078125, + "rewards/tag_count_reward/std": 0.2088298797607422, + "step": 1083 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1991.0, + "completions/mean_length": 777.044921875, + "completions/mean_terminated_length": 754.3041381835938, + "completions/min_length": 40.0, + "completions/min_terminated_length": 40.0, + "epoch": 0.3700605957156269, + "grad_norm": 0.945247232913971, + "kl": 5.203125, + "learning_rate": 8.145467336995954e-07, + "loss": 0.2983, + "num_tokens": 636637718.0, + "reward": 1.8046875, + "reward_std": 0.6311696767807007, + "rewards/accuracy_reward/mean": 0.123046875, + "rewards/accuracy_reward/std": 0.32881227135658264, + "rewards/format_reward/mean": 0.77734375, + "rewards/format_reward/std": 0.41643625497817993, + "rewards/tag_count_reward/mean": 0.904296875, + "rewards/tag_count_reward/std": 0.20836491882801056, + "step": 1084 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1994.0, + "completions/mean_length": 837.783203125, + "completions/mean_terminated_length": 806.2545166015625, + "completions/min_length": 98.0, + "completions/min_terminated_length": 98.0, + "epoch": 0.37040198002901764, + "grad_norm": 1.0525614023208618, + "kl": 5.484375, + "learning_rate": 8.141128631572676e-07, + "loss": 0.3226, + "num_tokens": 637144727.0, + "reward": 1.67578125, + "reward_std": 0.6332330703735352, + "rewards/accuracy_reward/mean": 0.0520833320915699, + "rewards/accuracy_reward/std": 0.2224269062280655, + "rewards/format_reward/mean": 0.736328125, + "rewards/format_reward/std": 0.4410543739795685, + "rewards/tag_count_reward/mean": 0.890625, + "rewards/tag_count_reward/std": 0.22338789701461792, + "step": 1085 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 905.400390625, + "completions/mean_terminated_length": 873.2791137695312, + "completions/min_length": 215.0, + "completions/min_terminated_length": 215.0, + "epoch": 0.37074336434240845, + "grad_norm": 0.8585146069526672, + "kl": 6.1640625, + "learning_rate": 8.136786177550373e-07, + "loss": 0.3761, + "num_tokens": 637686452.0, + "reward": 1.7041015625, + "reward_std": 0.6443427205085754, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.75390625, + "rewards/format_reward/std": 0.4311550557613373, + "rewards/tag_count_reward/mean": 0.8935546875, + "rewards/tag_count_reward/std": 0.22206437587738037, + "step": 1086 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1970.0, + "completions/mean_length": 863.5859375, + "completions/mean_terminated_length": 825.3790283203125, + "completions/min_length": 139.0, + "completions/min_terminated_length": 139.0, + "epoch": 0.37108474865579927, + "grad_norm": 0.929337203502655, + "kl": 4.8125, + "learning_rate": 8.132439981092364e-07, + "loss": 0.2587, + "num_tokens": 638208752.0, + "reward": 1.78369140625, + "reward_std": 0.6057051420211792, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.7890625, + "rewards/format_reward/std": 0.4083731174468994, + "rewards/tag_count_reward/mean": 0.91455078125, + "rewards/tag_count_reward/std": 0.201870396733284, + "step": 1087 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2011.0, + "completions/mean_length": 924.69921875, + "completions/mean_terminated_length": 879.0365600585938, + "completions/min_length": 203.0, + "completions/min_terminated_length": 203.0, + "epoch": 0.3714261329691901, + "grad_norm": 1.6980890035629272, + "kl": 4.96875, + "learning_rate": 8.128090048367283e-07, + "loss": 0.3049, + "num_tokens": 638757366.0, + "reward": 1.76806640625, + "reward_std": 0.602573573589325, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.787109375, + "rewards/format_reward/std": 0.409751296043396, + "rewards/tag_count_reward/mean": 0.91064453125, + "rewards/tag_count_reward/std": 0.20380185544490814, + "step": 1088 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1916.0, + "completions/mean_length": 821.224609375, + "completions/mean_terminated_length": 791.7820434570312, + "completions/min_length": 226.0, + "completions/min_terminated_length": 226.0, + "epoch": 0.37176751728258084, + "grad_norm": 1.3999513387680054, + "kl": 5.125, + "learning_rate": 8.123736385549063e-07, + "loss": 0.3134, + "num_tokens": 639254041.0, + "reward": 1.79443359375, + "reward_std": 0.5958268642425537, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.7890625, + "rewards/format_reward/std": 0.4083731174468994, + "rewards/tag_count_reward/mean": 0.91943359375, + "rewards/tag_count_reward/std": 0.1914980411529541, + "step": 1089 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 931.2734375, + "completions/mean_terminated_length": 888.2352905273438, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "epoch": 0.37210890159597165, + "grad_norm": 2.167673349380493, + "kl": 6.609375, + "learning_rate": 8.119378998816932e-07, + "loss": 0.3793, + "num_tokens": 639805157.0, + "reward": 1.75927734375, + "reward_std": 0.6634529232978821, + "rewards/accuracy_reward/mean": 0.125, + "rewards/accuracy_reward/std": 0.3310423493385315, + "rewards/format_reward/mean": 0.74609375, + "rewards/format_reward/std": 0.43567025661468506, + "rewards/tag_count_reward/mean": 0.88818359375, + "rewards/tag_count_reward/std": 0.22734324634075165, + "step": 1090 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2014.0, + "completions/mean_length": 860.47265625, + "completions/mean_terminated_length": 839.224609375, + "completions/min_length": 265.0, + "completions/min_terminated_length": 265.0, + "epoch": 0.37245028590936247, + "grad_norm": 0.7005177140235901, + "kl": 4.20703125, + "learning_rate": 8.115017894355401e-07, + "loss": 0.2119, + "num_tokens": 640327143.0, + "reward": 1.7822265625, + "reward_std": 0.571142852306366, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.787109375, + "rewards/format_reward/std": 0.409751296043396, + "rewards/tag_count_reward/mean": 0.9189453125, + "rewards/tag_count_reward/std": 0.1916109025478363, + "step": 1091 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 795.486328125, + "completions/mean_terminated_length": 767.9860229492188, + "completions/min_length": 211.0, + "completions/min_terminated_length": 211.0, + "epoch": 0.3727916702227533, + "grad_norm": 0.9756630659103394, + "kl": 4.40234375, + "learning_rate": 8.110653078354264e-07, + "loss": 0.2466, + "num_tokens": 640818288.0, + "reward": 1.75048828125, + "reward_std": 0.5791925191879272, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.7890625, + "rewards/format_reward/std": 0.4083731174468994, + "rewards/tag_count_reward/mean": 0.91455078125, + "rewards/tag_count_reward/std": 0.19634176790714264, + "step": 1092 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 886.421875, + "completions/mean_terminated_length": 824.27978515625, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "epoch": 0.37313305453614404, + "grad_norm": 1.5486931800842285, + "kl": 6.171875, + "learning_rate": 8.106284557008577e-07, + "loss": 0.3443, + "num_tokens": 641345560.0, + "reward": 1.70458984375, + "reward_std": 0.6492677927017212, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.744140625, + "rewards/format_reward/std": 0.43676990270614624, + "rewards/tag_count_reward/mean": 0.88818359375, + "rewards/tag_count_reward/std": 0.22626470029354095, + "step": 1093 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2019.0, + "completions/mean_length": 866.28125, + "completions/mean_terminated_length": 825.697021484375, + "completions/min_length": 187.0, + "completions/min_terminated_length": 187.0, + "epoch": 0.37347443884953485, + "grad_norm": 1.7973084449768066, + "kl": 5.1953125, + "learning_rate": 8.101912336518656e-07, + "loss": 0.3019, + "num_tokens": 641866584.0, + "reward": 1.755859375, + "reward_std": 0.6487219929695129, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.765625, + "rewards/format_reward/std": 0.42402184009552, + "rewards/tag_count_reward/mean": 0.904296875, + "rewards/tag_count_reward/std": 0.20718760788440704, + "step": 1094 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1851.0, + "completions/mean_length": 911.845703125, + "completions/mean_terminated_length": 865.6605224609375, + "completions/min_length": 292.0, + "completions/min_terminated_length": 292.0, + "epoch": 0.37381582316292566, + "grad_norm": 1.6846357583999634, + "kl": 4.26171875, + "learning_rate": 8.097536423090072e-07, + "loss": 0.2737, + "num_tokens": 642411369.0, + "reward": 1.79931640625, + "reward_std": 0.5785388946533203, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.779296875, + "rewards/format_reward/std": 0.4151262938976288, + "rewards/tag_count_reward/mean": 0.92236328125, + "rewards/tag_count_reward/std": 0.18756051361560822, + "step": 1095 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1925.0, + "completions/mean_length": 845.501953125, + "completions/mean_terminated_length": 786.3626708984375, + "completions/min_length": 19.0, + "completions/min_terminated_length": 19.0, + "epoch": 0.3741572074763165, + "grad_norm": 1.0550159215927124, + "kl": 6.0703125, + "learning_rate": 8.09315682293363e-07, + "loss": 0.3591, + "num_tokens": 642922058.0, + "reward": 1.71044921875, + "reward_std": 0.6264170408248901, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.759765625, + "rewards/format_reward/std": 0.4276435375213623, + "rewards/tag_count_reward/mean": 0.89208984375, + "rewards/tag_count_reward/std": 0.22491775453090668, + "step": 1096 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 874.115234375, + "completions/mean_terminated_length": 831.3421020507812, + "completions/min_length": 130.0, + "completions/min_terminated_length": 130.0, + "epoch": 0.37449859178970724, + "grad_norm": 1.7731609344482422, + "kl": 6.234375, + "learning_rate": 8.088773542265372e-07, + "loss": 0.365, + "num_tokens": 643437077.0, + "reward": 1.77880859375, + "reward_std": 0.625165581703186, + "rewards/accuracy_reward/mean": 0.109375, + "rewards/accuracy_reward/std": 0.31241437792778015, + "rewards/format_reward/mean": 0.771484375, + "rewards/format_reward/std": 0.4202871024608612, + "rewards/tag_count_reward/mean": 0.89794921875, + "rewards/tag_count_reward/std": 0.21887609362602234, + "step": 1097 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2007.0, + "completions/mean_length": 878.9140625, + "completions/mean_terminated_length": 826.4244384765625, + "completions/min_length": 9.0, + "completions/min_terminated_length": 9.0, + "epoch": 0.37483997610309805, + "grad_norm": 1.2832894325256348, + "kl": 5.66796875, + "learning_rate": 8.084386587306566e-07, + "loss": 0.3312, + "num_tokens": 643963929.0, + "reward": 1.74560546875, + "reward_std": 0.6264725923538208, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.755859375, + "rewards/format_reward/std": 0.42999663949012756, + "rewards/tag_count_reward/mean": 0.89599609375, + "rewards/tag_count_reward/std": 0.21907246112823486, + "step": 1098 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.060546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 907.818359375, + "completions/mean_terminated_length": 834.334716796875, + "completions/min_length": 144.0, + "completions/min_terminated_length": 144.0, + "epoch": 0.37518136041648886, + "grad_norm": 1.0897924900054932, + "kl": 7.1953125, + "learning_rate": 8.079995964283688e-07, + "loss": 0.4779, + "num_tokens": 644510588.0, + "reward": 1.6796875, + "reward_std": 0.6540594100952148, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.728515625, + "rewards/format_reward/std": 0.44516023993492126, + "rewards/tag_count_reward/mean": 0.876953125, + "rewards/tag_count_reward/std": 0.23091770708560944, + "step": 1099 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1932.0, + "completions/mean_length": 857.712890625, + "completions/mean_terminated_length": 811.8397216796875, + "completions/min_length": 177.0, + "completions/min_terminated_length": 177.0, + "epoch": 0.3755227447298797, + "grad_norm": 1.4612879753112793, + "kl": 5.4609375, + "learning_rate": 8.075601679428427e-07, + "loss": 0.3507, + "num_tokens": 645027049.0, + "reward": 1.69189453125, + "reward_std": 0.6441332101821899, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.748046875, + "rewards/format_reward/std": 0.43455907702445984, + "rewards/tag_count_reward/mean": 0.89501953125, + "rewards/tag_count_reward/std": 0.2152220904827118, + "step": 1100 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 815.390625, + "completions/mean_terminated_length": 775.6290283203125, + "completions/min_length": 167.0, + "completions/min_terminated_length": 167.0, + "epoch": 0.37586412904327043, + "grad_norm": 1.3878822326660156, + "kl": 5.3828125, + "learning_rate": 8.071203738977667e-07, + "loss": 0.3169, + "num_tokens": 645519089.0, + "reward": 1.75, + "reward_std": 0.6552772521972656, + "rewards/accuracy_reward/mean": 0.12708333134651184, + "rewards/accuracy_reward/std": 0.3334137797355652, + "rewards/format_reward/mean": 0.740234375, + "rewards/format_reward/std": 0.4389347732067108, + "rewards/tag_count_reward/mean": 0.890625, + "rewards/tag_count_reward/std": 0.2244802713394165, + "step": 1101 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 888.880859375, + "completions/mean_terminated_length": 839.3055419921875, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.37620551335666125, + "grad_norm": 3.209505796432495, + "kl": 6.3984375, + "learning_rate": 8.066802149173479e-07, + "loss": 0.3408, + "num_tokens": 646054292.0, + "reward": 1.58984375, + "reward_std": 0.704230546951294, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.669921875, + "rewards/format_reward/std": 0.47070086002349854, + "rewards/tag_count_reward/mean": 0.8515625, + "rewards/tag_count_reward/std": 0.2451835423707962, + "step": 1102 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1992.0, + "completions/mean_length": 791.171875, + "completions/mean_terminated_length": 745.3765258789062, + "completions/min_length": 101.0, + "completions/min_terminated_length": 101.0, + "epoch": 0.37654689767005206, + "grad_norm": 2.4412620067596436, + "kl": 5.25, + "learning_rate": 8.062396916263112e-07, + "loss": 0.3245, + "num_tokens": 646532140.0, + "reward": 1.7099609375, + "reward_std": 0.6323409676551819, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.748046875, + "rewards/format_reward/std": 0.43455907702445984, + "rewards/tag_count_reward/mean": 0.8974609375, + "rewards/tag_count_reward/std": 0.20921388268470764, + "step": 1103 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1930.0, + "completions/mean_length": 780.115234375, + "completions/mean_terminated_length": 752.2774658203125, + "completions/min_length": 4.0, + "completions/min_terminated_length": 4.0, + "epoch": 0.3768882819834429, + "grad_norm": 1.7001005411148071, + "kl": 5.140625, + "learning_rate": 8.057988046498993e-07, + "loss": 0.3435, + "num_tokens": 647011143.0, + "reward": 1.74462890625, + "reward_std": 0.6385067105293274, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.76953125, + "rewards/format_reward/std": 0.42154473066329956, + "rewards/tag_count_reward/mean": 0.90478515625, + "rewards/tag_count_reward/std": 0.20946665108203888, + "step": 1104 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1971.0, + "completions/mean_length": 832.859375, + "completions/mean_terminated_length": 801.202392578125, + "completions/min_length": 138.0, + "completions/min_terminated_length": 138.0, + "epoch": 0.3772296662968337, + "grad_norm": 2.0844802856445312, + "kl": 4.96875, + "learning_rate": 8.053575546138706e-07, + "loss": 0.3106, + "num_tokens": 647509087.0, + "reward": 1.6865234375, + "reward_std": 0.628883957862854, + "rewards/accuracy_reward/mean": 0.04233871027827263, + "rewards/accuracy_reward/std": 0.2015640139579773, + "rewards/format_reward/mean": 0.75390625, + "rewards/format_reward/std": 0.4311550557613373, + "rewards/tag_count_reward/mean": 0.8916015625, + "rewards/tag_count_reward/std": 0.22000661492347717, + "step": 1105 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 808.99609375, + "completions/mean_terminated_length": 763.8502197265625, + "completions/min_length": 15.0, + "completions/min_terminated_length": 15.0, + "epoch": 0.37757105061022445, + "grad_norm": 1.9999911785125732, + "kl": 6.65625, + "learning_rate": 8.049159421444986e-07, + "loss": 0.4405, + "num_tokens": 647994125.0, + "reward": 1.72216796875, + "reward_std": 0.6616029739379883, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.763671875, + "rewards/format_reward/std": 0.42524150013923645, + "rewards/tag_count_reward/mean": 0.89208984375, + "rewards/tag_count_reward/std": 0.22975978255271912, + "step": 1106 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1937.0, + "completions/mean_length": 903.98828125, + "completions/mean_terminated_length": 867.0846557617188, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.37791243492361526, + "grad_norm": 1.6477216482162476, + "kl": 5.2890625, + "learning_rate": 8.044739678685713e-07, + "loss": 0.2957, + "num_tokens": 648545143.0, + "reward": 1.7177734375, + "reward_std": 0.5774356126785278, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.759765625, + "rewards/format_reward/std": 0.4276435375213623, + "rewards/tag_count_reward/mean": 0.9150390625, + "rewards/tag_count_reward/std": 0.193101167678833, + "step": 1107 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1948.0, + "completions/mean_length": 830.173828125, + "completions/mean_terminated_length": 790.8890991210938, + "completions/min_length": 151.0, + "completions/min_terminated_length": 151.0, + "epoch": 0.3782538192370061, + "grad_norm": 2.0305397510528564, + "kl": 6.46875, + "learning_rate": 8.040316324133907e-07, + "loss": 0.3835, + "num_tokens": 649045344.0, + "reward": 1.8037109375, + "reward_std": 0.6847227215766907, + "rewards/accuracy_reward/mean": 0.123046875, + "rewards/accuracy_reward/std": 0.32881227135658264, + "rewards/format_reward/mean": 0.775390625, + "rewards/format_reward/std": 0.41773295402526855, + "rewards/tag_count_reward/mean": 0.9052734375, + "rewards/tag_count_reward/std": 0.20997978746891022, + "step": 1108 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.00390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2017.0, + "completions/mean_length": 719.869140625, + "completions/mean_terminated_length": 714.6608276367188, + "completions/min_length": 177.0, + "completions/min_terminated_length": 177.0, + "epoch": 0.3785952035503969, + "grad_norm": 1.0011018514633179, + "kl": 4.45703125, + "learning_rate": 8.035889364067709e-07, + "loss": 0.2692, + "num_tokens": 649489821.0, + "reward": 1.90478515625, + "reward_std": 0.5357630252838135, + "rewards/accuracy_reward/mean": 0.123046875, + "rewards/accuracy_reward/std": 0.32881227135658264, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.93798828125, + "rewards/tag_count_reward/std": 0.16971616446971893, + "step": 1109 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 806.626953125, + "completions/mean_terminated_length": 776.8340454101562, + "completions/min_length": 229.0, + "completions/min_terminated_length": 229.0, + "epoch": 0.37893658786378764, + "grad_norm": 2.0366158485412598, + "kl": 5.234375, + "learning_rate": 8.031458804770379e-07, + "loss": 0.3228, + "num_tokens": 649982926.0, + "reward": 1.8173828125, + "reward_std": 0.5892714262008667, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.80859375, + "rewards/format_reward/std": 0.3937928080558777, + "rewards/tag_count_reward/mean": 0.9248046875, + "rewards/tag_count_reward/std": 0.19016924500465393, + "step": 1110 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2016.0, + "completions/mean_length": 872.439453125, + "completions/mean_terminated_length": 809.5493774414062, + "completions/min_length": 35.0, + "completions/min_terminated_length": 35.0, + "epoch": 0.37927797217717846, + "grad_norm": 1.166548490524292, + "kl": 5.96875, + "learning_rate": 8.027024652530285e-07, + "loss": 0.3623, + "num_tokens": 650507407.0, + "reward": 1.775390625, + "reward_std": 0.6269536018371582, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.779296875, + "rewards/format_reward/std": 0.4151262938976288, + "rewards/tag_count_reward/mean": 0.90625, + "rewards/tag_count_reward/std": 0.20925270020961761, + "step": 1111 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2002.0, + "completions/mean_length": 744.90625, + "completions/mean_terminated_length": 724.2222900390625, + "completions/min_length": 177.0, + "completions/min_terminated_length": 177.0, + "epoch": 0.37961935649056927, + "grad_norm": 0.8393130898475647, + "kl": 5.328125, + "learning_rate": 8.022586913640896e-07, + "loss": 0.334, + "num_tokens": 650965071.0, + "reward": 1.85791015625, + "reward_std": 0.4956350326538086, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.92431640625, + "rewards/tag_count_reward/std": 0.18965290486812592, + "step": 1112 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1825.0, + "completions/mean_length": 801.142578125, + "completions/mean_terminated_length": 758.3212280273438, + "completions/min_length": 254.0, + "completions/min_terminated_length": 254.0, + "epoch": 0.3799607408039601, + "grad_norm": 1.9378595352172852, + "kl": 6.4921875, + "learning_rate": 8.018145594400772e-07, + "loss": 0.4415, + "num_tokens": 651452248.0, + "reward": 1.86279296875, + "reward_std": 0.6022834777832031, + "rewards/accuracy_reward/mean": 0.115234375, + "rewards/accuracy_reward/std": 0.3196168541908264, + "rewards/format_reward/mean": 0.828125, + "rewards/format_reward/std": 0.3776407241821289, + "rewards/tag_count_reward/mean": 0.91943359375, + "rewards/tag_count_reward/std": 0.19962850213050842, + "step": 1113 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1958.0, + "completions/mean_length": 818.1953125, + "completions/mean_terminated_length": 786.1563110351562, + "completions/min_length": 149.0, + "completions/min_terminated_length": 149.0, + "epoch": 0.38030212511735084, + "grad_norm": 3.2051782608032227, + "kl": 6.484375, + "learning_rate": 8.01370070111355e-07, + "loss": 0.3638, + "num_tokens": 651938412.0, + "reward": 1.802734375, + "reward_std": 0.5860434770584106, + "rewards/accuracy_reward/mean": 0.099609375, + "rewards/accuracy_reward/std": 0.29977133870124817, + "rewards/format_reward/mean": 0.79296875, + "rewards/format_reward/std": 0.40557438135147095, + "rewards/tag_count_reward/mean": 0.91015625, + "rewards/tag_count_reward/std": 0.1984144002199173, + "step": 1114 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2017.0, + "completions/mean_length": 722.341796875, + "completions/mean_terminated_length": 698.6222534179688, + "completions/min_length": 198.0, + "completions/min_terminated_length": 198.0, + "epoch": 0.38064350943074166, + "grad_norm": 1.0053163766860962, + "kl": 4.33203125, + "learning_rate": 8.009252240087947e-07, + "loss": 0.244, + "num_tokens": 652384875.0, + "reward": 1.8671875, + "reward_std": 0.5169427990913391, + "rewards/accuracy_reward/mean": 0.087890625, + "rewards/accuracy_reward/std": 0.2834126651287079, + "rewards/format_reward/mean": 0.841796875, + "rewards/format_reward/std": 0.36528825759887695, + "rewards/tag_count_reward/mean": 0.9375, + "rewards/tag_count_reward/std": 0.16478058695793152, + "step": 1115 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1942.0, + "completions/mean_length": 739.279296875, + "completions/mean_terminated_length": 705.1843872070312, + "completions/min_length": 157.0, + "completions/min_terminated_length": 157.0, + "epoch": 0.38098489374413247, + "grad_norm": 1.4230599403381348, + "kl": 5.28125, + "learning_rate": 8.004800217637736e-07, + "loss": 0.3337, + "num_tokens": 652837802.0, + "reward": 1.7919921875, + "reward_std": 0.5027107000350952, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.822265625, + "rewards/format_reward/std": 0.3826628625392914, + "rewards/tag_count_reward/mean": 0.9248046875, + "rewards/tag_count_reward/std": 0.18952499330043793, + "step": 1116 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1991.0, + "completions/mean_length": 814.55859375, + "completions/mean_terminated_length": 769.6154174804688, + "completions/min_length": 146.0, + "completions/min_terminated_length": 146.0, + "epoch": 0.3813262780575233, + "grad_norm": 2.010937452316284, + "kl": 5.62890625, + "learning_rate": 8.000344640081752e-07, + "loss": 0.3261, + "num_tokens": 653342456.0, + "reward": 1.69970703125, + "reward_std": 0.6149077415466309, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.74609375, + "rewards/format_reward/std": 0.43567025661468506, + "rewards/tag_count_reward/mean": 0.89501953125, + "rewards/tag_count_reward/std": 0.2135103940963745, + "step": 1117 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2018.0, + "completions/mean_length": 747.55859375, + "completions/mean_terminated_length": 713.6793823242188, + "completions/min_length": 164.0, + "completions/min_terminated_length": 164.0, + "epoch": 0.38166766237091404, + "grad_norm": 1.4754900932312012, + "kl": 5.3828125, + "learning_rate": 7.99588551374387e-07, + "loss": 0.366, + "num_tokens": 653798374.0, + "reward": 1.869140625, + "reward_std": 0.5730654001235962, + "rewards/accuracy_reward/mean": 0.109375, + "rewards/accuracy_reward/std": 0.31241437792778015, + "rewards/format_reward/mean": 0.833984375, + "rewards/format_reward/std": 0.3724585771560669, + "rewards/tag_count_reward/mean": 0.92578125, + "rewards/tag_count_reward/std": 0.18268859386444092, + "step": 1118 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1994.0, + "completions/mean_length": 774.439453125, + "completions/mean_terminated_length": 743.8740234375, + "completions/min_length": 35.0, + "completions/min_terminated_length": 35.0, + "epoch": 0.38200904668430485, + "grad_norm": 1.3872768878936768, + "kl": 5.78125, + "learning_rate": 7.991422844953004e-07, + "loss": 0.378, + "num_tokens": 654269255.0, + "reward": 1.75732421875, + "reward_std": 0.5963304042816162, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.794921875, + "rewards/format_reward/std": 0.4041535556316376, + "rewards/tag_count_reward/mean": 0.90966796875, + "rewards/tag_count_reward/std": 0.20034043490886688, + "step": 1119 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2021.0, + "completions/mean_length": 711.677734375, + "completions/mean_terminated_length": 676.8637084960938, + "completions/min_length": 99.0, + "completions/min_terminated_length": 99.0, + "epoch": 0.38235043099769567, + "grad_norm": 1.223705768585205, + "kl": 5.46484375, + "learning_rate": 7.986956640043096e-07, + "loss": 0.342, + "num_tokens": 654710114.0, + "reward": 1.85205078125, + "reward_std": 0.6124395132064819, + "rewards/accuracy_reward/mean": 0.119140625, + "rewards/accuracy_reward/std": 0.32427072525024414, + "rewards/format_reward/mean": 0.814453125, + "rewards/format_reward/std": 0.38912075757980347, + "rewards/tag_count_reward/mean": 0.91845703125, + "rewards/tag_count_reward/std": 0.19362683594226837, + "step": 1120 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1981.0, + "completions/mean_length": 857.865234375, + "completions/mean_terminated_length": 814.5, + "completions/min_length": 173.0, + "completions/min_terminated_length": 173.0, + "epoch": 0.3826918153110865, + "grad_norm": 0.999646008014679, + "kl": 5.953125, + "learning_rate": 7.98248690535311e-07, + "loss": 0.3592, + "num_tokens": 655231117.0, + "reward": 1.7109375, + "reward_std": 0.6151527166366577, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.751953125, + "rewards/format_reward/std": 0.4323015511035919, + "rewards/tag_count_reward/mean": 0.892578125, + "rewards/tag_count_reward/std": 0.21315263211727142, + "step": 1121 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2006.0, + "completions/mean_length": 786.4609375, + "completions/mean_terminated_length": 756.1840209960938, + "completions/min_length": 190.0, + "completions/min_terminated_length": 190.0, + "epoch": 0.38303319962447724, + "grad_norm": 1.6539634466171265, + "kl": 4.64453125, + "learning_rate": 7.978013647227015e-07, + "loss": 0.2793, + "num_tokens": 655705001.0, + "reward": 1.8173828125, + "reward_std": 0.6110129356384277, + "rewards/accuracy_reward/mean": 0.12298387289047241, + "rewards/accuracy_reward/std": 0.32875028252601624, + "rewards/format_reward/mean": 0.791015625, + "rewards/format_reward/std": 0.40698084235191345, + "rewards/tag_count_reward/mean": 0.9072265625, + "rewards/tag_count_reward/std": 0.2049688994884491, + "step": 1122 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.056640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2012.0, + "completions/mean_length": 886.181640625, + "completions/mean_terminated_length": 816.4244384765625, + "completions/min_length": 83.0, + "completions/min_terminated_length": 83.0, + "epoch": 0.38337458393786805, + "grad_norm": 1.147435188293457, + "kl": 6.3125, + "learning_rate": 7.973536872013783e-07, + "loss": 0.3892, + "num_tokens": 656237638.0, + "reward": 1.67529296875, + "reward_std": 0.6209849119186401, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.736328125, + "rewards/format_reward/std": 0.4410543739795685, + "rewards/tag_count_reward/mean": 0.88427734375, + "rewards/tag_count_reward/std": 0.22537609934806824, + "step": 1123 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2041.0, + "completions/mean_length": 798.8828125, + "completions/mean_terminated_length": 755.98388671875, + "completions/min_length": 139.0, + "completions/min_terminated_length": 139.0, + "epoch": 0.38371596825125887, + "grad_norm": 2.6214802265167236, + "kl": 6.09375, + "learning_rate": 7.969056586067376e-07, + "loss": 0.4175, + "num_tokens": 656716842.0, + "reward": 1.7373046875, + "reward_std": 0.6505454778671265, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.771484375, + "rewards/format_reward/std": 0.4202871024608612, + "rewards/tag_count_reward/mean": 0.8876953125, + "rewards/tag_count_reward/std": 0.22247707843780518, + "step": 1124 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1992.0, + "completions/mean_length": 783.39453125, + "completions/mean_terminated_length": 737.3157958984375, + "completions/min_length": 114.0, + "completions/min_terminated_length": 114.0, + "epoch": 0.3840573525646497, + "grad_norm": 1.1501455307006836, + "kl": 6.0234375, + "learning_rate": 7.964572795746741e-07, + "loss": 0.3478, + "num_tokens": 657195524.0, + "reward": 1.712890625, + "reward_std": 0.6205965280532837, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.7578125, + "rewards/format_reward/std": 0.42882615327835083, + "rewards/tag_count_reward/mean": 0.896484375, + "rewards/tag_count_reward/std": 0.21790531277656555, + "step": 1125 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 775.951171875, + "completions/mean_terminated_length": 729.6012573242188, + "completions/min_length": 35.0, + "completions/min_terminated_length": 35.0, + "epoch": 0.38439873687804044, + "grad_norm": 1.1001150608062744, + "kl": 5.9609375, + "learning_rate": 7.960085507415802e-07, + "loss": 0.3623, + "num_tokens": 657662219.0, + "reward": 1.6796875, + "reward_std": 0.6390104293823242, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.734375, + "rewards/format_reward/std": 0.44209739565849304, + "rewards/tag_count_reward/mean": 0.892578125, + "rewards/tag_count_reward/std": 0.21825583279132843, + "step": 1126 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2015.0, + "completions/mean_length": 785.125, + "completions/mean_terminated_length": 733.78857421875, + "completions/min_length": 133.0, + "completions/min_terminated_length": 133.0, + "epoch": 0.38474012119143125, + "grad_norm": 1.8485504388809204, + "kl": 5.0859375, + "learning_rate": 7.955594727443439e-07, + "loss": 0.3444, + "num_tokens": 658146091.0, + "reward": 1.81298828125, + "reward_std": 0.643386960029602, + "rewards/accuracy_reward/mean": 0.138671875, + "rewards/accuracy_reward/std": 0.34594178199768066, + "rewards/format_reward/mean": 0.771484375, + "rewards/format_reward/std": 0.4202871024608612, + "rewards/tag_count_reward/mean": 0.90283203125, + "rewards/tag_count_reward/std": 0.21089871227741241, + "step": 1127 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2023.0, + "completions/mean_length": 788.6328125, + "completions/mean_terminated_length": 748.008056640625, + "completions/min_length": 150.0, + "completions/min_terminated_length": 150.0, + "epoch": 0.38508150550482206, + "grad_norm": 1.121274709701538, + "kl": 6.5390625, + "learning_rate": 7.951100462203494e-07, + "loss": 0.3914, + "num_tokens": 658626671.0, + "reward": 1.7099609375, + "reward_std": 0.6317379474639893, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.75390625, + "rewards/format_reward/std": 0.4311550557613373, + "rewards/tag_count_reward/mean": 0.8974609375, + "rewards/tag_count_reward/std": 0.21326690912246704, + "step": 1128 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1994.0, + "completions/mean_length": 779.080078125, + "completions/mean_terminated_length": 735.5010375976562, + "completions/min_length": 173.0, + "completions/min_terminated_length": 173.0, + "epoch": 0.3854228898182129, + "grad_norm": 1.466565728187561, + "kl": 5.6484375, + "learning_rate": 7.946602718074756e-07, + "loss": 0.3559, + "num_tokens": 659094696.0, + "reward": 1.72265625, + "reward_std": 0.604473352432251, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.76953125, + "rewards/format_reward/std": 0.42154473066329956, + "rewards/tag_count_reward/mean": 0.91015625, + "rewards/tag_count_reward/std": 0.20147298276424408, + "step": 1129 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2005.0, + "completions/mean_length": 848.818359375, + "completions/mean_terminated_length": 802.6024169921875, + "completions/min_length": 99.0, + "completions/min_terminated_length": 99.0, + "epoch": 0.38576427413160363, + "grad_norm": 1.2333524227142334, + "kl": 6.3125, + "learning_rate": 7.94210150144095e-07, + "loss": 0.3712, + "num_tokens": 659616603.0, + "reward": 1.7236328125, + "reward_std": 0.6640071868896484, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.7578125, + "rewards/format_reward/std": 0.42882615327835083, + "rewards/tag_count_reward/mean": 0.8876953125, + "rewards/tag_count_reward/std": 0.2284444123506546, + "step": 1130 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 793.828125, + "completions/mean_terminated_length": 750.755615234375, + "completions/min_length": 171.0, + "completions/min_terminated_length": 171.0, + "epoch": 0.38610565844499445, + "grad_norm": 1.7627885341644287, + "kl": 5.8515625, + "learning_rate": 7.937596818690729e-07, + "loss": 0.3843, + "num_tokens": 660099267.0, + "reward": 1.755859375, + "reward_std": 0.5984328389167786, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.767578125, + "rewards/format_reward/std": 0.42278963327407837, + "rewards/tag_count_reward/mean": 0.90234375, + "rewards/tag_count_reward/std": 0.2062724232673645, + "step": 1131 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.056640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 851.26953125, + "completions/mean_terminated_length": 779.4161376953125, + "completions/min_length": 119.0, + "completions/min_terminated_length": 119.0, + "epoch": 0.38644704275838526, + "grad_norm": 1.089171051979065, + "kl": 7.8046875, + "learning_rate": 7.933088676217667e-07, + "loss": 0.4961, + "num_tokens": 660611261.0, + "reward": 1.72021484375, + "reward_std": 0.6671257019042969, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.751953125, + "rewards/format_reward/std": 0.4323015511035919, + "rewards/tag_count_reward/mean": 0.88623046875, + "rewards/tag_count_reward/std": 0.22637026011943817, + "step": 1132 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 847.890625, + "completions/mean_terminated_length": 783.6871948242188, + "completions/min_length": 160.0, + "completions/min_terminated_length": 160.0, + "epoch": 0.3867884270717761, + "grad_norm": 1.5207698345184326, + "kl": 6.5, + "learning_rate": 7.928577080420247e-07, + "loss": 0.3852, + "num_tokens": 661132309.0, + "reward": 1.7197265625, + "reward_std": 0.6027827262878418, + "rewards/accuracy_reward/mean": 0.052419353276491165, + "rewards/accuracy_reward/std": 0.22309619188308716, + "rewards/format_reward/mean": 0.771484375, + "rewards/format_reward/std": 0.4202871024608612, + "rewards/tag_count_reward/mean": 0.8974609375, + "rewards/tag_count_reward/std": 0.2189268320798874, + "step": 1133 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2023.0, + "completions/mean_length": 809.419921875, + "completions/mean_terminated_length": 756.446044921875, + "completions/min_length": 145.0, + "completions/min_terminated_length": 145.0, + "epoch": 0.38712981138516683, + "grad_norm": 1.6754188537597656, + "kl": 6.68359375, + "learning_rate": 7.924062037701853e-07, + "loss": 0.3849, + "num_tokens": 661626140.0, + "reward": 1.75048828125, + "reward_std": 0.6199695467948914, + "rewards/accuracy_reward/mean": 0.099609375, + "rewards/accuracy_reward/std": 0.29977133870124817, + "rewards/format_reward/mean": 0.7578125, + "rewards/format_reward/std": 0.42882615327835083, + "rewards/tag_count_reward/mean": 0.89306640625, + "rewards/tag_count_reward/std": 0.22538457810878754, + "step": 1134 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1951.0, + "completions/mean_length": 838.259765625, + "completions/mean_terminated_length": 783.9448852539062, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "epoch": 0.38747119569855765, + "grad_norm": 4.741856098175049, + "kl": 6.609375, + "learning_rate": 7.919543554470763e-07, + "loss": 0.433, + "num_tokens": 662139985.0, + "reward": 1.736328125, + "reward_std": 0.6018377542495728, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.7734375, + "rewards/format_reward/std": 0.4190165400505066, + "rewards/tag_count_reward/mean": 0.900390625, + "rewards/tag_count_reward/std": 0.2146535962820053, + "step": 1135 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 834.3125, + "completions/mean_terminated_length": 797.6820678710938, + "completions/min_length": 21.0, + "completions/min_terminated_length": 21.0, + "epoch": 0.38781258001194846, + "grad_norm": 3.1156985759735107, + "kl": 5.0703125, + "learning_rate": 7.91502163714014e-07, + "loss": 0.3285, + "num_tokens": 662650705.0, + "reward": 1.7841796875, + "reward_std": 0.6189864873886108, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.79296875, + "rewards/format_reward/std": 0.40557438135147095, + "rewards/tag_count_reward/mean": 0.9130859375, + "rewards/tag_count_reward/std": 0.20575061440467834, + "step": 1136 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2005.0, + "completions/mean_length": 784.7890625, + "completions/mean_terminated_length": 741.4060668945312, + "completions/min_length": 52.0, + "completions/min_terminated_length": 52.0, + "epoch": 0.3881539643253393, + "grad_norm": 1.924870252609253, + "kl": 5.875, + "learning_rate": 7.910496292128015e-07, + "loss": 0.3487, + "num_tokens": 663132341.0, + "reward": 1.78173828125, + "reward_std": 0.6319122314453125, + "rewards/accuracy_reward/mean": 0.126953125, + "rewards/accuracy_reward/std": 0.33324605226516724, + "rewards/format_reward/mean": 0.76171875, + "rewards/format_reward/std": 0.42644867300987244, + "rewards/tag_count_reward/mean": 0.89306640625, + "rewards/tag_count_reward/std": 0.21539516746997833, + "step": 1137 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1965.0, + "completions/mean_length": 856.783203125, + "completions/mean_terminated_length": 820.8309326171875, + "completions/min_length": 61.0, + "completions/min_terminated_length": 61.0, + "epoch": 0.38849534863873003, + "grad_norm": 2.1889028549194336, + "kl": 5.28125, + "learning_rate": 7.905967525857291e-07, + "loss": 0.3409, + "num_tokens": 663646950.0, + "reward": 1.8046875, + "reward_std": 0.621527373790741, + "rewards/accuracy_reward/mean": 0.111328125, + "rewards/accuracy_reward/std": 0.31484565138816833, + "rewards/format_reward/mean": 0.78515625, + "rewards/format_reward/std": 0.4111155867576599, + "rewards/tag_count_reward/mean": 0.908203125, + "rewards/tag_count_reward/std": 0.20659643411636353, + "step": 1138 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2016.0, + "completions/mean_length": 818.861328125, + "completions/mean_terminated_length": 766.291259765625, + "completions/min_length": 147.0, + "completions/min_terminated_length": 147.0, + "epoch": 0.38883673295212084, + "grad_norm": 3.511672019958496, + "kl": 7.4609375, + "learning_rate": 7.901435344755721e-07, + "loss": 0.3917, + "num_tokens": 664136751.0, + "reward": 1.646484375, + "reward_std": 0.6125409007072449, + "rewards/accuracy_reward/mean": 0.03427419438958168, + "rewards/accuracy_reward/std": 0.18211629986763, + "rewards/format_reward/mean": 0.728515625, + "rewards/format_reward/std": 0.44516023993492126, + "rewards/tag_count_reward/mean": 0.884765625, + "rewards/tag_count_reward/std": 0.21250611543655396, + "step": 1139 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 795.7109375, + "completions/mean_terminated_length": 747.4482421875, + "completions/min_length": 5.0, + "completions/min_terminated_length": 5.0, + "epoch": 0.38917811726551166, + "grad_norm": 4.841424942016602, + "kl": 7.9375, + "learning_rate": 7.896899755255906e-07, + "loss": 0.4211, + "num_tokens": 664622091.0, + "reward": 1.669921875, + "reward_std": 0.6799638867378235, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.732421875, + "rewards/format_reward/std": 0.4431293308734894, + "rewards/tag_count_reward/mean": 0.865234375, + "rewards/tag_count_reward/std": 0.23177681863307953, + "step": 1140 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2025.0, + "completions/mean_length": 779.025390625, + "completions/mean_terminated_length": 732.7874755859375, + "completions/min_length": 176.0, + "completions/min_terminated_length": 176.0, + "epoch": 0.38951950157890247, + "grad_norm": 1.623910665512085, + "kl": 7.03125, + "learning_rate": 7.892360763795291e-07, + "loss": 0.4307, + "num_tokens": 665096840.0, + "reward": 1.64697265625, + "reward_std": 0.6152798533439636, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.712890625, + "rewards/format_reward/std": 0.45285552740097046, + "rewards/tag_count_reward/mean": 0.88134765625, + "rewards/tag_count_reward/std": 0.2160595804452896, + "step": 1141 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2006.0, + "completions/mean_length": 815.537109375, + "completions/mean_terminated_length": 749.6028442382812, + "completions/min_length": 10.0, + "completions/min_terminated_length": 10.0, + "epoch": 0.38986088589229323, + "grad_norm": 2.9458398818969727, + "kl": 7.3125, + "learning_rate": 7.887818376816136e-07, + "loss": 0.426, + "num_tokens": 665586011.0, + "reward": 1.67724609375, + "reward_std": 0.6621624231338501, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.7109375, + "rewards/format_reward/std": 0.45377036929130554, + "rewards/tag_count_reward/mean": 0.88037109375, + "rewards/tag_count_reward/std": 0.22332319617271423, + "step": 1142 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 778.068359375, + "completions/mean_terminated_length": 734.45458984375, + "completions/min_length": 118.0, + "completions/min_terminated_length": 118.0, + "epoch": 0.39020227020568404, + "grad_norm": 1.1650549173355103, + "kl": 5.578125, + "learning_rate": 7.883272600765535e-07, + "loss": 0.3751, + "num_tokens": 666067230.0, + "reward": 1.73388671875, + "reward_std": 0.7144436836242676, + "rewards/accuracy_reward/mean": 0.138671875, + "rewards/accuracy_reward/std": 0.34594178199768066, + "rewards/format_reward/mean": 0.71875, + "rewards/format_reward/std": 0.45004892349243164, + "rewards/tag_count_reward/mean": 0.87646484375, + "rewards/tag_count_reward/std": 0.2271834760904312, + "step": 1143 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0703125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2009.0, + "completions/mean_length": 880.03515625, + "completions/mean_terminated_length": 791.7017211914062, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "epoch": 0.39054365451907486, + "grad_norm": 3.26562237739563, + "kl": 5.5546875, + "learning_rate": 7.878723442095384e-07, + "loss": 0.3861, + "num_tokens": 666607184.0, + "reward": 1.67236328125, + "reward_std": 0.6616606116294861, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.728515625, + "rewards/format_reward/std": 0.44516023993492126, + "rewards/tag_count_reward/mean": 0.87158203125, + "rewards/tag_count_reward/std": 0.23037032783031464, + "step": 1144 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1862.0, + "completions/mean_length": 775.494140625, + "completions/mean_terminated_length": 739.7208862304688, + "completions/min_length": 32.0, + "completions/min_terminated_length": 32.0, + "epoch": 0.39088503883246567, + "grad_norm": 2.17079496383667, + "kl": 4.13671875, + "learning_rate": 7.87417090726238e-07, + "loss": 0.2229, + "num_tokens": 667082557.0, + "reward": 1.68994140625, + "reward_std": 0.5987516045570374, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.75390625, + "rewards/format_reward/std": 0.4311550557613373, + "rewards/tag_count_reward/mean": 0.89501953125, + "rewards/tag_count_reward/std": 0.2059287577867508, + "step": 1145 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 770.181640625, + "completions/mean_terminated_length": 726.2969970703125, + "completions/min_length": 4.0, + "completions/min_terminated_length": 4.0, + "epoch": 0.39122642314585643, + "grad_norm": 2.750588893890381, + "kl": 4.109375, + "learning_rate": 7.869615002728016e-07, + "loss": 0.2714, + "num_tokens": 667559306.0, + "reward": 1.7763671875, + "reward_std": 0.6300910711288452, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.77734375, + "rewards/format_reward/std": 0.41643625497817993, + "rewards/tag_count_reward/mean": 0.9013671875, + "rewards/tag_count_reward/std": 0.20992517471313477, + "step": 1146 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2024.0, + "completions/mean_length": 759.791015625, + "completions/mean_terminated_length": 736.7415161132812, + "completions/min_length": 40.0, + "completions/min_terminated_length": 40.0, + "epoch": 0.39156780745924724, + "grad_norm": 1.8268723487854004, + "kl": 4.4921875, + "learning_rate": 7.865055734958566e-07, + "loss": 0.2689, + "num_tokens": 668020495.0, + "reward": 1.75439453125, + "reward_std": 0.6093716621398926, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.78125, + "rewards/format_reward/std": 0.41380295157432556, + "rewards/tag_count_reward/mean": 0.90673828125, + "rewards/tag_count_reward/std": 0.1958935260772705, + "step": 1147 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2020.0, + "completions/mean_length": 779.1953125, + "completions/mean_terminated_length": 751.3373413085938, + "completions/min_length": 154.0, + "completions/min_terminated_length": 154.0, + "epoch": 0.39190919177263805, + "grad_norm": 1.9029971361160278, + "kl": 4.5703125, + "learning_rate": 7.860493110425073e-07, + "loss": 0.2848, + "num_tokens": 668499283.0, + "reward": 1.79443359375, + "reward_std": 0.5710021257400513, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.796875, + "rewards/format_reward/std": 0.4027182459831238, + "rewards/tag_count_reward/mean": 0.91552734375, + "rewards/tag_count_reward/std": 0.18915848433971405, + "step": 1148 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 802.76171875, + "completions/mean_terminated_length": 767.7550048828125, + "completions/min_length": 170.0, + "completions/min_terminated_length": 170.0, + "epoch": 0.39225057608602887, + "grad_norm": 1.7392467260360718, + "kl": 5.77734375, + "learning_rate": 7.855927135603348e-07, + "loss": 0.3378, + "num_tokens": 668987689.0, + "reward": 1.82861328125, + "reward_std": 0.6556215882301331, + "rewards/accuracy_reward/mean": 0.142578125, + "rewards/accuracy_reward/std": 0.3499840497970581, + "rewards/format_reward/mean": 0.77734375, + "rewards/format_reward/std": 0.41643625497817993, + "rewards/tag_count_reward/mean": 0.90869140625, + "rewards/tag_count_reward/std": 0.1892089992761612, + "step": 1149 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2005.0, + "completions/mean_length": 771.7578125, + "completions/mean_terminated_length": 722.5719604492188, + "completions/min_length": 107.0, + "completions/min_terminated_length": 107.0, + "epoch": 0.3925919603994196, + "grad_norm": 2.6620638370513916, + "kl": 5.5546875, + "learning_rate": 7.851357816973962e-07, + "loss": 0.2986, + "num_tokens": 669455725.0, + "reward": 1.7470703125, + "reward_std": 0.6239137053489685, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.76171875, + "rewards/format_reward/std": 0.42644867300987244, + "rewards/tag_count_reward/mean": 0.8994140625, + "rewards/tag_count_reward/std": 0.20604762434959412, + "step": 1150 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2009.0, + "completions/mean_length": 842.150390625, + "completions/mean_terminated_length": 790.576416015625, + "completions/min_length": 8.0, + "completions/min_terminated_length": 8.0, + "epoch": 0.39293334471281044, + "grad_norm": 3.0081570148468018, + "kl": 6.8984375, + "learning_rate": 7.846785161022223e-07, + "loss": 0.3995, + "num_tokens": 669958394.0, + "reward": 1.734375, + "reward_std": 0.6649159789085388, + "rewards/accuracy_reward/mean": 0.09879032522439957, + "rewards/accuracy_reward/std": 0.2986815273761749, + "rewards/format_reward/mean": 0.74609375, + "rewards/format_reward/std": 0.43567025661468506, + "rewards/tag_count_reward/mean": 0.892578125, + "rewards/tag_count_reward/std": 0.21257804334163666, + "step": 1151 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1985.0, + "completions/mean_length": 847.01171875, + "completions/mean_terminated_length": 787.9466552734375, + "completions/min_length": 5.0, + "completions/min_terminated_length": 5.0, + "epoch": 0.39327472902620125, + "grad_norm": 3.3438761234283447, + "kl": 7.7421875, + "learning_rate": 7.842209174238181e-07, + "loss": 0.4798, + "num_tokens": 670475520.0, + "reward": 1.6552734375, + "reward_std": 0.6544797420501709, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.71875, + "rewards/format_reward/std": 0.45004892349243164, + "rewards/tag_count_reward/mean": 0.8798828125, + "rewards/tag_count_reward/std": 0.23034387826919556, + "step": 1152 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2009.0, + "completions/mean_length": 768.03515625, + "completions/mean_terminated_length": 745.1331787109375, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.39361611333959207, + "grad_norm": 2.016155242919922, + "kl": 5.3203125, + "learning_rate": 7.837629863116612e-07, + "loss": 0.2692, + "num_tokens": 670951250.0, + "reward": 1.78955078125, + "reward_std": 0.5713357925415039, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.796875, + "rewards/format_reward/std": 0.4027182459831238, + "rewards/tag_count_reward/mean": 0.91259765625, + "rewards/tag_count_reward/std": 0.18976370990276337, + "step": 1153 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 866.634765625, + "completions/mean_terminated_length": 816.1079711914062, + "completions/min_length": 30.0, + "completions/min_terminated_length": 30.0, + "epoch": 0.3939574976529828, + "grad_norm": 2.2790541648864746, + "kl": 6.4453125, + "learning_rate": 7.833047234157012e-07, + "loss": 0.3606, + "num_tokens": 671463255.0, + "reward": 1.638671875, + "reward_std": 0.6134622097015381, + "rewards/accuracy_reward/mean": 0.0463709682226181, + "rewards/accuracy_reward/std": 0.21049949526786804, + "rewards/format_reward/mean": 0.71875, + "rewards/format_reward/std": 0.45004892349243164, + "rewards/tag_count_reward/mean": 0.875, + "rewards/tag_count_reward/std": 0.2314550280570984, + "step": 1154 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 788.60546875, + "completions/mean_terminated_length": 737.4105224609375, + "completions/min_length": 172.0, + "completions/min_terminated_length": 172.0, + "epoch": 0.39429888196637364, + "grad_norm": 1.8915692567825317, + "kl": 5.0078125, + "learning_rate": 7.828461293863581e-07, + "loss": 0.3307, + "num_tokens": 671943085.0, + "reward": 1.77685546875, + "reward_std": 0.6183324456214905, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.77734375, + "rewards/format_reward/std": 0.41643625497817993, + "rewards/tag_count_reward/mean": 0.90576171875, + "rewards/tag_count_reward/std": 0.2045973688364029, + "step": 1155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 827.72265625, + "completions/mean_terminated_length": 780.6936645507812, + "completions/min_length": 233.0, + "completions/min_terminated_length": 233.0, + "epoch": 0.39464026627976445, + "grad_norm": 4.631453514099121, + "kl": 4.27734375, + "learning_rate": 7.823872048745223e-07, + "loss": 0.297, + "num_tokens": 672447455.0, + "reward": 1.72265625, + "reward_std": 0.6078072190284729, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.779296875, + "rewards/format_reward/std": 0.4151262938976288, + "rewards/tag_count_reward/mean": 0.8984375, + "rewards/tag_count_reward/std": 0.21714095771312714, + "step": 1156 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2020.0, + "completions/mean_length": 812.89453125, + "completions/mean_terminated_length": 783.2520141601562, + "completions/min_length": 239.0, + "completions/min_terminated_length": 239.0, + "epoch": 0.39498165059315526, + "grad_norm": 4.364314079284668, + "kl": 4.75, + "learning_rate": 7.819279505315538e-07, + "loss": 0.3488, + "num_tokens": 672949945.0, + "reward": 1.68310546875, + "reward_std": 0.6080790758132935, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.734375, + "rewards/format_reward/std": 0.44209739565849304, + "rewards/tag_count_reward/mean": 0.89013671875, + "rewards/tag_count_reward/std": 0.2195560783147812, + "step": 1157 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1900.0, + "completions/mean_length": 804.609375, + "completions/mean_terminated_length": 756.6896362304688, + "completions/min_length": 158.0, + "completions/min_terminated_length": 158.0, + "epoch": 0.395323034906546, + "grad_norm": 3.419736385345459, + "kl": 4.8515625, + "learning_rate": 7.814683670092795e-07, + "loss": 0.3505, + "num_tokens": 673437345.0, + "reward": 1.72314453125, + "reward_std": 0.5805853605270386, + "rewards/accuracy_reward/mean": 0.029296875, + "rewards/accuracy_reward/std": 0.16880230605602264, + "rewards/format_reward/mean": 0.79296875, + "rewards/format_reward/std": 0.40557438135147095, + "rewards/tag_count_reward/mean": 0.90087890625, + "rewards/tag_count_reward/std": 0.21287843585014343, + "step": 1158 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 783.759765625, + "completions/mean_terminated_length": 737.6943359375, + "completions/min_length": 148.0, + "completions/min_terminated_length": 148.0, + "epoch": 0.39566441921993684, + "grad_norm": 0.9830715656280518, + "kl": 5.828125, + "learning_rate": 7.810084549599944e-07, + "loss": 0.3455, + "num_tokens": 673911862.0, + "reward": 1.68310546875, + "reward_std": 0.6620784997940063, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.24230584502220154, + "rewards/format_reward/mean": 0.73828125, + "rewards/format_reward/std": 0.44000017642974854, + "rewards/tag_count_reward/mean": 0.88427734375, + "rewards/tag_count_reward/std": 0.22537609934806824, + "step": 1159 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 905.68359375, + "completions/mean_terminated_length": 854.3958740234375, + "completions/min_length": 182.0, + "completions/min_terminated_length": 182.0, + "epoch": 0.39600580353332765, + "grad_norm": 1.4845670461654663, + "kl": 7.3125, + "learning_rate": 7.805482150364598e-07, + "loss": 0.4588, + "num_tokens": 674459092.0, + "reward": 1.64453125, + "reward_std": 0.6831563711166382, + "rewards/accuracy_reward/mean": 0.08064515888690948, + "rewards/accuracy_reward/std": 0.2725643217563629, + "rewards/format_reward/mean": 0.705078125, + "rewards/format_reward/std": 0.4564536213874817, + "rewards/tag_count_reward/mean": 0.861328125, + "rewards/tag_count_reward/std": 0.24741044640541077, + "step": 1160 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1991.0, + "completions/mean_length": 775.59375, + "completions/mean_terminated_length": 726.5557861328125, + "completions/min_length": 14.0, + "completions/min_terminated_length": 14.0, + "epoch": 0.39634718784671846, + "grad_norm": 1.3064024448394775, + "kl": 7.1484375, + "learning_rate": 7.800876478919014e-07, + "loss": 0.4384, + "num_tokens": 674928212.0, + "reward": 1.6708984375, + "reward_std": 0.6619834899902344, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.720703125, + "rewards/format_reward/std": 0.44909247756004333, + "rewards/tag_count_reward/mean": 0.8798828125, + "rewards/tag_count_reward/std": 0.22223642468452454, + "step": 1161 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2025.0, + "completions/mean_length": 840.60546875, + "completions/mean_terminated_length": 788.9653930664062, + "completions/min_length": 240.0, + "completions/min_terminated_length": 240.0, + "epoch": 0.3966885721601092, + "grad_norm": 2.027768135070801, + "kl": 6.3515625, + "learning_rate": 7.796267541800106e-07, + "loss": 0.3414, + "num_tokens": 675440698.0, + "reward": 1.67822265625, + "reward_std": 0.6405354738235474, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.71875, + "rewards/format_reward/std": 0.45004892349243164, + "rewards/tag_count_reward/mean": 0.88330078125, + "rewards/tag_count_reward/std": 0.22757430374622345, + "step": 1162 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1988.0, + "completions/mean_length": 863.26953125, + "completions/mean_terminated_length": 805.0040283203125, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "epoch": 0.39702995647350003, + "grad_norm": 0.8648762702941895, + "kl": 6.640625, + "learning_rate": 7.791655345549416e-07, + "loss": 0.411, + "num_tokens": 675958212.0, + "reward": 1.68798828125, + "reward_std": 0.6656864285469055, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.7265625, + "rewards/format_reward/std": 0.4461594223976135, + "rewards/tag_count_reward/mean": 0.87744140625, + "rewards/tag_count_reward/std": 0.23666778206825256, + "step": 1163 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 789.015625, + "completions/mean_terminated_length": 743.1417236328125, + "completions/min_length": 120.0, + "completions/min_terminated_length": 120.0, + "epoch": 0.39737134078689085, + "grad_norm": 2.4499430656433105, + "kl": 5.78125, + "learning_rate": 7.78703989671311e-07, + "loss": 0.3827, + "num_tokens": 676439676.0, + "reward": 1.72216796875, + "reward_std": 0.6407417058944702, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.75390625, + "rewards/format_reward/std": 0.4311550557613373, + "rewards/tag_count_reward/mean": 0.89599609375, + "rewards/tag_count_reward/std": 0.21851344406604767, + "step": 1164 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2023.0, + "completions/mean_length": 775.470703125, + "completions/mean_terminated_length": 742.3186645507812, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.39771272510028166, + "grad_norm": 2.001849889755249, + "kl": 4.80078125, + "learning_rate": 7.782421201841978e-07, + "loss": 0.277, + "num_tokens": 676905613.0, + "reward": 1.78955078125, + "reward_std": 0.5900582075119019, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.80859375, + "rewards/format_reward/std": 0.3937928080558777, + "rewards/tag_count_reward/mean": 0.91259765625, + "rewards/tag_count_reward/std": 0.2004214972257614, + "step": 1165 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2010.0, + "completions/mean_length": 866.25, + "completions/mean_terminated_length": 810.6666259765625, + "completions/min_length": 226.0, + "completions/min_terminated_length": 226.0, + "epoch": 0.3980541094136724, + "grad_norm": 1.3362925052642822, + "kl": 6.625, + "learning_rate": 7.77779926749141e-07, + "loss": 0.3893, + "num_tokens": 677425021.0, + "reward": 1.66748046875, + "reward_std": 0.6531873941421509, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.740234375, + "rewards/format_reward/std": 0.4389347732067108, + "rewards/tag_count_reward/mean": 0.88818359375, + "rewards/tag_count_reward/std": 0.21912479400634766, + "step": 1166 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 846.55859375, + "completions/mean_terminated_length": 790.049072265625, + "completions/min_length": 109.0, + "completions/min_terminated_length": 109.0, + "epoch": 0.39839549372706323, + "grad_norm": 1.2076319456100464, + "kl": 6.578125, + "learning_rate": 7.773174100221398e-07, + "loss": 0.4258, + "num_tokens": 677936427.0, + "reward": 1.7744140625, + "reward_std": 0.6134333610534668, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.78515625, + "rewards/format_reward/std": 0.4111155867576599, + "rewards/tag_count_reward/mean": 0.9033203125, + "rewards/tag_count_reward/std": 0.2159915715456009, + "step": 1167 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2030.0, + "completions/mean_length": 836.38671875, + "completions/mean_terminated_length": 799.8189086914062, + "completions/min_length": 142.0, + "completions/min_terminated_length": 142.0, + "epoch": 0.39873687804045405, + "grad_norm": 1.737146019935608, + "kl": 6.1015625, + "learning_rate": 7.768545706596519e-07, + "loss": 0.3958, + "num_tokens": 678439377.0, + "reward": 1.75732421875, + "reward_std": 0.5887085199356079, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.810546875, + "rewards/format_reward/std": 0.3922513723373413, + "rewards/tag_count_reward/mean": 0.90185546875, + "rewards/tag_count_reward/std": 0.22121235728263855, + "step": 1168 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1962.0, + "completions/mean_length": 825.546875, + "completions/mean_terminated_length": 788.65185546875, + "completions/min_length": 177.0, + "completions/min_terminated_length": 177.0, + "epoch": 0.39907826235384486, + "grad_norm": 1.1133967638015747, + "kl": 6.1640625, + "learning_rate": 7.763914093185932e-07, + "loss": 0.3867, + "num_tokens": 678936105.0, + "reward": 1.8193359375, + "reward_std": 0.5887194871902466, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.814453125, + "rewards/format_reward/std": 0.38912075757980347, + "rewards/tag_count_reward/mean": 0.9130859375, + "rewards/tag_count_reward/std": 0.20335890352725983, + "step": 1169 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1937.0, + "completions/mean_length": 848.7578125, + "completions/mean_terminated_length": 794.9142456054688, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "epoch": 0.3994196466672356, + "grad_norm": 2.6224238872528076, + "kl": 7.09375, + "learning_rate": 7.759279266563365e-07, + "loss": 0.3943, + "num_tokens": 679445325.0, + "reward": 1.7685546875, + "reward_std": 0.6187150478363037, + "rewards/accuracy_reward/mean": 0.08467742055654526, + "rewards/accuracy_reward/std": 0.278682142496109, + "rewards/format_reward/mean": 0.78515625, + "rewards/format_reward/std": 0.4111155867576599, + "rewards/tag_count_reward/mean": 0.9013671875, + "rewards/tag_count_reward/std": 0.21050699055194855, + "step": 1170 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2030.0, + "completions/mean_length": 787.0078125, + "completions/mean_terminated_length": 751.5582275390625, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.39976103098062643, + "grad_norm": 3.6764135360717773, + "kl": 8.1484375, + "learning_rate": 7.754641233307109e-07, + "loss": 0.4696, + "num_tokens": 679928801.0, + "reward": 1.724609375, + "reward_std": 0.6647849678993225, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.763671875, + "rewards/format_reward/std": 0.42524150013923645, + "rewards/tag_count_reward/mean": 0.884765625, + "rewards/tag_count_reward/std": 0.2275160402059555, + "step": 1171 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 796.189453125, + "completions/mean_terminated_length": 745.3027954101562, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "epoch": 0.40010241529401724, + "grad_norm": 3.195155620574951, + "kl": 8.125, + "learning_rate": 7.75e-07, + "loss": 0.4878, + "num_tokens": 680419746.0, + "reward": 1.748046875, + "reward_std": 0.5978066325187683, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.24230584502220154, + "rewards/format_reward/mean": 0.783203125, + "rewards/format_reward/std": 0.4124660789966583, + "rewards/tag_count_reward/mean": 0.904296875, + "rewards/tag_count_reward/std": 0.21358256042003632, + "step": 1172 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1926.0, + "completions/mean_length": 852.396484375, + "completions/mean_terminated_length": 818.7850952148438, + "completions/min_length": 148.0, + "completions/min_terminated_length": 148.0, + "epoch": 0.40044379960740806, + "grad_norm": 2.44873046875, + "kl": 7.609375, + "learning_rate": 7.745355573229422e-07, + "loss": 0.4326, + "num_tokens": 680937485.0, + "reward": 1.74169921875, + "reward_std": 0.6479066610336304, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.775390625, + "rewards/format_reward/std": 0.41773295402526855, + "rewards/tag_count_reward/mean": 0.90380859375, + "rewards/tag_count_reward/std": 0.2096034586429596, + "step": 1173 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1924.0, + "completions/mean_length": 786.751953125, + "completions/mean_terminated_length": 753.893798828125, + "completions/min_length": 38.0, + "completions/min_terminated_length": 38.0, + "epoch": 0.4007851839207988, + "grad_norm": 1.5025852918624878, + "kl": 6.1875, + "learning_rate": 7.740707959587289e-07, + "loss": 0.346, + "num_tokens": 681412910.0, + "reward": 1.81494140625, + "reward_std": 0.6183757781982422, + "rewards/accuracy_reward/mean": 0.109375, + "rewards/accuracy_reward/std": 0.31241437792778015, + "rewards/format_reward/mean": 0.79296875, + "rewards/format_reward/std": 0.40557438135147095, + "rewards/tag_count_reward/mean": 0.91259765625, + "rewards/tag_count_reward/std": 0.2058405727148056, + "step": 1174 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2024.0, + "completions/mean_length": 818.048828125, + "completions/mean_terminated_length": 786.0060424804688, + "completions/min_length": 17.0, + "completions/min_terminated_length": 17.0, + "epoch": 0.40112656823418963, + "grad_norm": 2.9225752353668213, + "kl": 4.734375, + "learning_rate": 7.736057165670038e-07, + "loss": 0.2795, + "num_tokens": 681906071.0, + "reward": 1.763671875, + "reward_std": 0.5842170715332031, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.806640625, + "rewards/format_reward/std": 0.39531853795051575, + "rewards/tag_count_reward/mean": 0.916015625, + "rewards/tag_count_reward/std": 0.19852031767368317, + "step": 1175 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1981.0, + "completions/mean_length": 901.255859375, + "completions/mean_terminated_length": 839.9074096679688, + "completions/min_length": 109.0, + "completions/min_terminated_length": 109.0, + "epoch": 0.40146795254758044, + "grad_norm": 1.8672685623168945, + "kl": 5.1953125, + "learning_rate": 7.731403198078622e-07, + "loss": 0.3463, + "num_tokens": 682441722.0, + "reward": 1.70166015625, + "reward_std": 0.608293354511261, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.76953125, + "rewards/format_reward/std": 0.42154473066329956, + "rewards/tag_count_reward/mean": 0.89111328125, + "rewards/tag_count_reward/std": 0.22225522994995117, + "step": 1176 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 847.671875, + "completions/mean_terminated_length": 823.760986328125, + "completions/min_length": 152.0, + "completions/min_terminated_length": 152.0, + "epoch": 0.40180933686097126, + "grad_norm": 3.887122631072998, + "kl": 3.65234375, + "learning_rate": 7.726746063418493e-07, + "loss": 0.2463, + "num_tokens": 682950146.0, + "reward": 1.8203125, + "reward_std": 0.5349379777908325, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.828125, + "rewards/format_reward/std": 0.3776407241821289, + "rewards/tag_count_reward/mean": 0.927734375, + "rewards/tag_count_reward/std": 0.17528937757015228, + "step": 1177 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 901.046875, + "completions/mean_terminated_length": 829.6597900390625, + "completions/min_length": 194.0, + "completions/min_terminated_length": 194.0, + "epoch": 0.402150721174362, + "grad_norm": 1.545910358428955, + "kl": 6.703125, + "learning_rate": 7.722085768299608e-07, + "loss": 0.4523, + "num_tokens": 683491658.0, + "reward": 1.6962890625, + "reward_std": 0.620068371295929, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.748046875, + "rewards/format_reward/std": 0.43455907702445984, + "rewards/tag_count_reward/mean": 0.8759765625, + "rewards/tag_count_reward/std": 0.23719459772109985, + "step": 1178 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1986.0, + "completions/mean_length": 910.20703125, + "completions/mean_terminated_length": 839.3900756835938, + "completions/min_length": 48.0, + "completions/min_terminated_length": 48.0, + "epoch": 0.4024921054877528, + "grad_norm": 1.2167710065841675, + "kl": 6.359375, + "learning_rate": 7.717422319336398e-07, + "loss": 0.3604, + "num_tokens": 684033188.0, + "reward": 1.66552734375, + "reward_std": 0.6231434345245361, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.736328125, + "rewards/format_reward/std": 0.4410543739795685, + "rewards/tag_count_reward/mean": 0.87255859375, + "rewards/tag_count_reward/std": 0.23459148406982422, + "step": 1179 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1965.0, + "completions/mean_length": 857.51171875, + "completions/mean_terminated_length": 788.6404418945312, + "completions/min_length": 220.0, + "completions/min_terminated_length": 220.0, + "epoch": 0.40283348980114364, + "grad_norm": 1.1367852687835693, + "kl": 5.72265625, + "learning_rate": 7.712755723147777e-07, + "loss": 0.3528, + "num_tokens": 684550506.0, + "reward": 1.73779296875, + "reward_std": 0.5876256227493286, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.775390625, + "rewards/format_reward/std": 0.41773295402526855, + "rewards/tag_count_reward/mean": 0.90185546875, + "rewards/tag_count_reward/std": 0.21218155324459076, + "step": 1180 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 803.064453125, + "completions/mean_terminated_length": 765.4909057617188, + "completions/min_length": 205.0, + "completions/min_terminated_length": 205.0, + "epoch": 0.40317487411453445, + "grad_norm": 2.600008964538574, + "kl": 5.7890625, + "learning_rate": 7.708085986357127e-07, + "loss": 0.4062, + "num_tokens": 685030939.0, + "reward": 1.81005859375, + "reward_std": 0.5699402689933777, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.814453125, + "rewards/format_reward/std": 0.38912075757980347, + "rewards/tag_count_reward/mean": 0.91943359375, + "rewards/tag_count_reward/std": 0.19085827469825745, + "step": 1181 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 826.353515625, + "completions/mean_terminated_length": 774.1038818359375, + "completions/min_length": 15.0, + "completions/min_terminated_length": 15.0, + "epoch": 0.4035162584279252, + "grad_norm": 1.3960096836090088, + "kl": 6.8046875, + "learning_rate": 7.703413115592282e-07, + "loss": 0.4225, + "num_tokens": 685530832.0, + "reward": 1.6708984375, + "reward_std": 0.6258046627044678, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.736328125, + "rewards/format_reward/std": 0.4410543739795685, + "rewards/tag_count_reward/mean": 0.8916015625, + "rewards/tag_count_reward/std": 0.2188919186592102, + "step": 1182 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2016.0, + "completions/mean_length": 852.904296875, + "completions/mean_terminated_length": 814.352783203125, + "completions/min_length": 18.0, + "completions/min_terminated_length": 18.0, + "epoch": 0.403857642741316, + "grad_norm": 2.728266477584839, + "kl": 7.3125, + "learning_rate": 7.69873711748553e-07, + "loss": 0.4203, + "num_tokens": 686046911.0, + "reward": 1.6982421875, + "reward_std": 0.6647390127182007, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.72265625, + "rewards/format_reward/std": 0.4481254518032074, + "rewards/tag_count_reward/mean": 0.8818359375, + "rewards/tag_count_reward/std": 0.22654588520526886, + "step": 1183 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 816.693359375, + "completions/mean_terminated_length": 761.41015625, + "completions/min_length": 149.0, + "completions/min_terminated_length": 149.0, + "epoch": 0.40419902705470684, + "grad_norm": 3.1669857501983643, + "kl": 7.453125, + "learning_rate": 7.694057998673597e-07, + "loss": 0.4283, + "num_tokens": 686544418.0, + "reward": 1.67333984375, + "reward_std": 0.7041476964950562, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.70703125, + "rewards/format_reward/std": 0.455569326877594, + "rewards/tag_count_reward/mean": 0.87451171875, + "rewards/tag_count_reward/std": 0.22772540152072906, + "step": 1184 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1923.0, + "completions/mean_length": 786.220703125, + "completions/mean_terminated_length": 753.3486938476562, + "completions/min_length": 119.0, + "completions/min_terminated_length": 119.0, + "epoch": 0.40454041136809765, + "grad_norm": 1.4147940874099731, + "kl": 4.953125, + "learning_rate": 7.68937576579763e-07, + "loss": 0.2801, + "num_tokens": 687030563.0, + "reward": 1.7041015625, + "reward_std": 0.6220543384552002, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.74609375, + "rewards/format_reward/std": 0.43567025661468506, + "rewards/tag_count_reward/mean": 0.8955078125, + "rewards/tag_count_reward/std": 0.2094147503376007, + "step": 1185 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 775.755859375, + "completions/mean_terminated_length": 734.7156982421875, + "completions/min_length": 171.0, + "completions/min_terminated_length": 171.0, + "epoch": 0.4048817956814884, + "grad_norm": 2.470407485961914, + "kl": 5.203125, + "learning_rate": 7.684690425503208e-07, + "loss": 0.3382, + "num_tokens": 687506742.0, + "reward": 1.76171875, + "reward_std": 0.5721713304519653, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.7734375, + "rewards/format_reward/std": 0.4190165400505066, + "rewards/tag_count_reward/mean": 0.908203125, + "rewards/tag_count_reward/std": 0.19997821748256683, + "step": 1186 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2023.0, + "completions/mean_length": 759.37109375, + "completions/mean_terminated_length": 725.7996215820312, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "epoch": 0.4052231799948792, + "grad_norm": 1.7808350324630737, + "kl": 4.9609375, + "learning_rate": 7.680001984440312e-07, + "loss": 0.3089, + "num_tokens": 687975044.0, + "reward": 1.77392578125, + "reward_std": 0.6071911454200745, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.7734375, + "rewards/format_reward/std": 0.4190165400505066, + "rewards/tag_count_reward/mean": 0.90478515625, + "rewards/tag_count_reward/std": 0.20294204354286194, + "step": 1187 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1973.0, + "completions/mean_length": 792.6484375, + "completions/mean_terminated_length": 733.603271484375, + "completions/min_length": 134.0, + "completions/min_terminated_length": 134.0, + "epoch": 0.40556456430827004, + "grad_norm": 1.2724409103393555, + "kl": 6.765625, + "learning_rate": 7.675310449263326e-07, + "loss": 0.42, + "num_tokens": 688453536.0, + "reward": 1.671875, + "reward_std": 0.6647064685821533, + "rewards/accuracy_reward/mean": 0.05416666716337204, + "rewards/accuracy_reward/std": 0.22658243775367737, + "rewards/format_reward/mean": 0.734375, + "rewards/format_reward/std": 0.44209739565849304, + "rewards/tag_count_reward/mean": 0.88671875, + "rewards/tag_count_reward/std": 0.21696485579013824, + "step": 1188 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1993.0, + "completions/mean_length": 827.001953125, + "completions/mean_terminated_length": 774.7800903320312, + "completions/min_length": 201.0, + "completions/min_terminated_length": 201.0, + "epoch": 0.40590594862166085, + "grad_norm": 1.154120683670044, + "kl": 6.21484375, + "learning_rate": 7.670615826631027e-07, + "loss": 0.3925, + "num_tokens": 688956433.0, + "reward": 1.74755859375, + "reward_std": 0.6169498562812805, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.763671875, + "rewards/format_reward/std": 0.42524150013923645, + "rewards/tag_count_reward/mean": 0.89013671875, + "rewards/tag_count_reward/std": 0.22989697754383087, + "step": 1189 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2006.0, + "completions/mean_length": 839.15625, + "completions/mean_terminated_length": 810.14404296875, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "epoch": 0.4062473329350516, + "grad_norm": 1.295357584953308, + "kl": 5.26953125, + "learning_rate": 7.665918123206572e-07, + "loss": 0.3226, + "num_tokens": 689454913.0, + "reward": 1.77294921875, + "reward_std": 0.5832884311676025, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.78515625, + "rewards/format_reward/std": 0.4111155867576599, + "rewards/tag_count_reward/mean": 0.91162109375, + "rewards/tag_count_reward/std": 0.19999195635318756, + "step": 1190 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1758.0, + "completions/mean_length": 767.30859375, + "completions/mean_terminated_length": 728.6558837890625, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "epoch": 0.4065887172484424, + "grad_norm": 2.6538870334625244, + "kl": 5.484375, + "learning_rate": 7.661217345657495e-07, + "loss": 0.3574, + "num_tokens": 689918015.0, + "reward": 1.81005859375, + "reward_std": 0.6497660279273987, + "rewards/accuracy_reward/mean": 0.1015625, + "rewards/accuracy_reward/std": 0.30236753821372986, + "rewards/format_reward/mean": 0.80078125, + "rewards/format_reward/std": 0.39980348944664, + "rewards/tag_count_reward/mean": 0.90771484375, + "rewards/tag_count_reward/std": 0.21019525825977325, + "step": 1191 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2009.0, + "completions/mean_length": 833.564453125, + "completions/mean_terminated_length": 768.5946044921875, + "completions/min_length": 210.0, + "completions/min_terminated_length": 210.0, + "epoch": 0.40693010156183324, + "grad_norm": 1.9779895544052124, + "kl": 6.890625, + "learning_rate": 7.656513500655688e-07, + "loss": 0.4104, + "num_tokens": 690418768.0, + "reward": 1.75927734375, + "reward_std": 0.5864324569702148, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.787109375, + "rewards/format_reward/std": 0.409751296043396, + "rewards/tag_count_reward/mean": 0.89404296875, + "rewards/tag_count_reward/std": 0.23066876828670502, + "step": 1192 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2023.0, + "completions/mean_length": 828.177734375, + "completions/mean_terminated_length": 781.1663208007812, + "completions/min_length": 99.0, + "completions/min_terminated_length": 99.0, + "epoch": 0.40727148587522405, + "grad_norm": 0.8671897053718567, + "kl": 6.2265625, + "learning_rate": 7.651806594877397e-07, + "loss": 0.3636, + "num_tokens": 690916587.0, + "reward": 1.74658203125, + "reward_std": 0.6254327297210693, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.77734375, + "rewards/format_reward/std": 0.41643625497817993, + "rewards/tag_count_reward/mean": 0.89697265625, + "rewards/tag_count_reward/std": 0.22394701838493347, + "step": 1193 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1842.0, + "completions/mean_length": 785.36328125, + "completions/mean_terminated_length": 736.7017822265625, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "epoch": 0.4076128701886148, + "grad_norm": 1.2734403610229492, + "kl": 6.390625, + "learning_rate": 7.647096635003216e-07, + "loss": 0.3881, + "num_tokens": 691391349.0, + "reward": 1.810546875, + "reward_std": 0.6113005876541138, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.8125, + "rewards/format_reward/std": 0.39069411158561707, + "rewards/tag_count_reward/mean": 0.908203125, + "rewards/tag_count_reward/std": 0.20718760788440704, + "step": 1194 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 870.53515625, + "completions/mean_terminated_length": 822.6707153320312, + "completions/min_length": 272.0, + "completions/min_terminated_length": 272.0, + "epoch": 0.4079542545020056, + "grad_norm": 2.4065842628479004, + "kl": 6.4453125, + "learning_rate": 7.642383627718072e-07, + "loss": 0.4508, + "num_tokens": 691920327.0, + "reward": 1.802734375, + "reward_std": 0.5818156003952026, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.81640625, + "rewards/format_reward/std": 0.3875311613082886, + "rewards/tag_count_reward/mean": 0.91015625, + "rewards/tag_count_reward/std": 0.21326914429664612, + "step": 1195 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2018.0, + "completions/mean_length": 776.951171875, + "completions/mean_terminated_length": 717.1676635742188, + "completions/min_length": 262.0, + "completions/min_terminated_length": 262.0, + "epoch": 0.40829563881539643, + "grad_norm": 1.3699885606765747, + "kl": 6.4375, + "learning_rate": 7.637667579711215e-07, + "loss": 0.4272, + "num_tokens": 692395326.0, + "reward": 1.8544921875, + "reward_std": 0.5869347453117371, + "rewards/accuracy_reward/mean": 0.099609375, + "rewards/accuracy_reward/std": 0.29977133870124817, + "rewards/format_reward/mean": 0.83203125, + "rewards/format_reward/std": 0.374204158782959, + "rewards/tag_count_reward/mean": 0.9228515625, + "rewards/tag_count_reward/std": 0.20006181299686432, + "step": 1196 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2009.0, + "completions/mean_length": 853.14453125, + "completions/mean_terminated_length": 794.381103515625, + "completions/min_length": 298.0, + "completions/min_terminated_length": 298.0, + "epoch": 0.40863702312878725, + "grad_norm": 1.542723536491394, + "kl": 6.453125, + "learning_rate": 7.632948497676213e-07, + "loss": 0.3742, + "num_tokens": 692917336.0, + "reward": 1.77783203125, + "reward_std": 0.6210418343544006, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.791015625, + "rewards/format_reward/std": 0.40698084235191345, + "rewards/tag_count_reward/mean": 0.90087890625, + "rewards/tag_count_reward/std": 0.2168629914522171, + "step": 1197 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.068359375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2033.0, + "completions/mean_length": 871.080078125, + "completions/mean_terminated_length": 784.7232666015625, + "completions/min_length": 109.0, + "completions/min_terminated_length": 109.0, + "epoch": 0.408978407442178, + "grad_norm": 1.8710319995880127, + "kl": 7.984375, + "learning_rate": 7.62822638831094e-07, + "loss": 0.4869, + "num_tokens": 693438993.0, + "reward": 1.7080078125, + "reward_std": 0.5856927037239075, + "rewards/accuracy_reward/mean": 0.024193547666072845, + "rewards/accuracy_reward/std": 0.15380479395389557, + "rewards/format_reward/mean": 0.7890625, + "rewards/format_reward/std": 0.4083731174468994, + "rewards/tag_count_reward/mean": 0.8955078125, + "rewards/tag_count_reward/std": 0.2321290373802185, + "step": 1198 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1769.0, + "completions/mean_length": 763.900390625, + "completions/mean_terminated_length": 748.6739501953125, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "epoch": 0.4093197917555688, + "grad_norm": 1.8708481788635254, + "kl": 3.703125, + "learning_rate": 7.623501258317567e-07, + "loss": 0.2097, + "num_tokens": 693899566.0, + "reward": 1.88134765625, + "reward_std": 0.5026733875274658, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.93994140625, + "rewards/tag_count_reward/std": 0.1732655018568039, + "step": 1199 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1914.0, + "completions/mean_length": 834.232421875, + "completions/mean_terminated_length": 782.31982421875, + "completions/min_length": 165.0, + "completions/min_terminated_length": 165.0, + "epoch": 0.40966117606895963, + "grad_norm": 1.1944559812545776, + "kl": 7.3984375, + "learning_rate": 7.618773114402554e-07, + "loss": 0.4642, + "num_tokens": 694406853.0, + "reward": 1.7294921875, + "reward_std": 0.5911108255386353, + "rewards/accuracy_reward/mean": 0.03515625, + "rewards/accuracy_reward/std": 0.1843547374010086, + "rewards/format_reward/mean": 0.791015625, + "rewards/format_reward/std": 0.40698084235191345, + "rewards/tag_count_reward/mean": 0.9033203125, + "rewards/tag_count_reward/std": 0.21824489533901215, + "step": 1200 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 777.416015625, + "completions/mean_terminated_length": 725.7662353515625, + "completions/min_length": 175.0, + "completions/min_terminated_length": 175.0, + "epoch": 0.41000256038235044, + "grad_norm": 1.3154793977737427, + "kl": 7.0703125, + "learning_rate": 7.61404196327663e-07, + "loss": 0.4424, + "num_tokens": 694879066.0, + "reward": 1.80615234375, + "reward_std": 0.5888994932174683, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.810546875, + "rewards/format_reward/std": 0.3922513723373413, + "rewards/tag_count_reward/mean": 0.90576171875, + "rewards/tag_count_reward/std": 0.21280211210250854, + "step": 1201 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1995.0, + "completions/mean_length": 830.302734375, + "completions/mean_terminated_length": 778.2220458984375, + "completions/min_length": 164.0, + "completions/min_terminated_length": 164.0, + "epoch": 0.41034394469574126, + "grad_norm": 1.958497405052185, + "kl": 6.9296875, + "learning_rate": 7.609307811654804e-07, + "loss": 0.4531, + "num_tokens": 695380469.0, + "reward": 1.75, + "reward_std": 0.55804443359375, + "rewards/accuracy_reward/mean": 0.029296875, + "rewards/accuracy_reward/std": 0.16880230605602264, + "rewards/format_reward/mean": 0.810546875, + "rewards/format_reward/std": 0.3922513723373413, + "rewards/tag_count_reward/mean": 0.91015625, + "rewards/tag_count_reward/std": 0.2062724232673645, + "step": 1202 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2031.0, + "completions/mean_length": 885.5234375, + "completions/mean_terminated_length": 825.8480834960938, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "epoch": 0.410685329009132, + "grad_norm": 1.5910775661468506, + "kl": 7.1796875, + "learning_rate": 7.604570666256339e-07, + "loss": 0.4556, + "num_tokens": 695915137.0, + "reward": 1.73779296875, + "reward_std": 0.5948113203048706, + "rewards/accuracy_reward/mean": 0.03125, + "rewards/accuracy_reward/std": 0.17416280508041382, + "rewards/format_reward/mean": 0.802734375, + "rewards/format_reward/std": 0.3983237147331238, + "rewards/tag_count_reward/mean": 0.90380859375, + "rewards/tag_count_reward/std": 0.2226203829050064, + "step": 1203 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1994.0, + "completions/mean_length": 792.06640625, + "completions/mean_terminated_length": 748.933349609375, + "completions/min_length": 187.0, + "completions/min_terminated_length": 187.0, + "epoch": 0.41102671332252283, + "grad_norm": 1.8395967483520508, + "kl": 6.5546875, + "learning_rate": 7.599830533804741e-07, + "loss": 0.383, + "num_tokens": 696399315.0, + "reward": 1.81103515625, + "reward_std": 0.5429419875144958, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.810546875, + "rewards/format_reward/std": 0.3922513723373413, + "rewards/tag_count_reward/mean": 0.90869140625, + "rewards/tag_count_reward/std": 0.21635127067565918, + "step": 1204 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1975.0, + "completions/mean_length": 717.984375, + "completions/mean_terminated_length": 691.4900512695312, + "completions/min_length": 166.0, + "completions/min_terminated_length": 166.0, + "epoch": 0.41136809763591364, + "grad_norm": 2.9843039512634277, + "kl": 4.9140625, + "learning_rate": 7.595087421027767e-07, + "loss": 0.3451, + "num_tokens": 696845227.0, + "reward": 1.87109375, + "reward_std": 0.5197564363479614, + "rewards/accuracy_reward/mean": 0.07459677755832672, + "rewards/accuracy_reward/std": 0.263004869222641, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.9375, + "rewards/tag_count_reward/std": 0.1863754689693451, + "step": 1205 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2021.0, + "completions/mean_length": 796.98046875, + "completions/mean_terminated_length": 759.2233276367188, + "completions/min_length": 237.0, + "completions/min_terminated_length": 237.0, + "epoch": 0.41170948194930446, + "grad_norm": 4.940004348754883, + "kl": 5.8359375, + "learning_rate": 7.590341334657395e-07, + "loss": 0.4393, + "num_tokens": 697329937.0, + "reward": 1.87255859375, + "reward_std": 0.5778836011886597, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.92919921875, + "rewards/tag_count_reward/std": 0.180350661277771, + "step": 1206 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1849.0, + "completions/mean_length": 856.20703125, + "completions/mean_terminated_length": 802.6979370117188, + "completions/min_length": 183.0, + "completions/min_terminated_length": 183.0, + "epoch": 0.4120508662626952, + "grad_norm": 0.8065212965011597, + "kl": 6.578125, + "learning_rate": 7.585592281429828e-07, + "loss": 0.3917, + "num_tokens": 697847083.0, + "reward": 1.7763671875, + "reward_std": 0.5780097842216492, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.81640625, + "rewards/format_reward/std": 0.3875311613082886, + "rewards/tag_count_reward/mean": 0.9091796875, + "rewards/tag_count_reward/std": 0.21227891743183136, + "step": 1207 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1947.0, + "completions/mean_length": 801.88671875, + "completions/mean_terminated_length": 751.231689453125, + "completions/min_length": 29.0, + "completions/min_terminated_length": 29.0, + "epoch": 0.41239225057608603, + "grad_norm": 1.0168187618255615, + "kl": 5.6796875, + "learning_rate": 7.580840268085477e-07, + "loss": 0.3418, + "num_tokens": 698337825.0, + "reward": 1.84130859375, + "reward_std": 0.5921512842178345, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.82421875, + "rewards/format_reward/std": 0.3810062110424042, + "rewards/tag_count_reward/mean": 0.91162109375, + "rewards/tag_count_reward/std": 0.2089642435312271, + "step": 1208 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2024.0, + "completions/mean_length": 880.62109375, + "completions/mean_terminated_length": 825.7136840820312, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.41273363488947684, + "grad_norm": 1.1118146181106567, + "kl": 8.015625, + "learning_rate": 7.576085301368955e-07, + "loss": 0.4832, + "num_tokens": 698865359.0, + "reward": 1.73193359375, + "reward_std": 0.6110692024230957, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.79296875, + "rewards/format_reward/std": 0.40557438135147095, + "rewards/tag_count_reward/mean": 0.89404296875, + "rewards/tag_count_reward/std": 0.23278005421161652, + "step": 1209 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 883.099609375, + "completions/mean_terminated_length": 818.24951171875, + "completions/min_length": 5.0, + "completions/min_terminated_length": 5.0, + "epoch": 0.41307501920286765, + "grad_norm": 3.1092116832733154, + "kl": 8.0234375, + "learning_rate": 7.571327388029071e-07, + "loss": 0.4615, + "num_tokens": 699387554.0, + "reward": 1.7412109375, + "reward_std": 0.6016433238983154, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.794921875, + "rewards/format_reward/std": 0.4041535556316376, + "rewards/tag_count_reward/mean": 0.8955078125, + "rewards/tag_count_reward/std": 0.22733746469020844, + "step": 1210 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2019.0, + "completions/mean_length": 806.70703125, + "completions/mean_terminated_length": 753.6171264648438, + "completions/min_length": 160.0, + "completions/min_terminated_length": 160.0, + "epoch": 0.4134164035162584, + "grad_norm": 1.6269659996032715, + "kl": 6.5859375, + "learning_rate": 7.566566534818809e-07, + "loss": 0.4204, + "num_tokens": 699887196.0, + "reward": 1.73388671875, + "reward_std": 0.6248112916946411, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.775390625, + "rewards/format_reward/std": 0.41773295402526855, + "rewards/tag_count_reward/mean": 0.89990234375, + "rewards/tag_count_reward/std": 0.21471090614795685, + "step": 1211 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1992.0, + "completions/mean_length": 829.37890625, + "completions/mean_terminated_length": 779.8414306640625, + "completions/min_length": 179.0, + "completions/min_terminated_length": 179.0, + "epoch": 0.4137577878296492, + "grad_norm": 1.2652716636657715, + "kl": 6.37109375, + "learning_rate": 7.561802748495332e-07, + "loss": 0.4199, + "num_tokens": 700390206.0, + "reward": 1.81884765625, + "reward_std": 0.6061054468154907, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.81640625, + "rewards/format_reward/std": 0.3875311613082886, + "rewards/tag_count_reward/mean": 0.91064453125, + "rewards/tag_count_reward/std": 0.20499862730503082, + "step": 1212 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2041.0, + "completions/mean_length": 827.82421875, + "completions/mean_terminated_length": 778.2235717773438, + "completions/min_length": 217.0, + "completions/min_terminated_length": 217.0, + "epoch": 0.41409917214304004, + "grad_norm": 1.3812679052352905, + "kl": 7.4765625, + "learning_rate": 7.557036035819963e-07, + "loss": 0.4555, + "num_tokens": 700891732.0, + "reward": 1.73193359375, + "reward_std": 0.6351964473724365, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.7734375, + "rewards/format_reward/std": 0.4190165400505066, + "rewards/tag_count_reward/mean": 0.89013671875, + "rewards/tag_count_reward/std": 0.23359142243862152, + "step": 1213 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1998.0, + "completions/mean_length": 819.130859375, + "completions/mean_terminated_length": 797.1431274414062, + "completions/min_length": 180.0, + "completions/min_terminated_length": 180.0, + "epoch": 0.41444055645643085, + "grad_norm": 1.9653494358062744, + "kl": 4.7265625, + "learning_rate": 7.552266403558176e-07, + "loss": 0.2734, + "num_tokens": 701386487.0, + "reward": 1.75830078125, + "reward_std": 0.5999512672424316, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.806640625, + "rewards/format_reward/std": 0.39531853795051575, + "rewards/tag_count_reward/mean": 0.90673828125, + "rewards/tag_count_reward/std": 0.21380557119846344, + "step": 1214 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 822.66796875, + "completions/mean_terminated_length": 754.45361328125, + "completions/min_length": 212.0, + "completions/min_terminated_length": 212.0, + "epoch": 0.4147819407698216, + "grad_norm": 1.7280266284942627, + "kl": 8.40625, + "learning_rate": 7.547493858479595e-07, + "loss": 0.5356, + "num_tokens": 701879565.0, + "reward": 1.724609375, + "reward_std": 0.7054948806762695, + "rewards/accuracy_reward/mean": 0.111328125, + "rewards/accuracy_reward/std": 0.31484565138816833, + "rewards/format_reward/mean": 0.740234375, + "rewards/format_reward/std": 0.4389347732067108, + "rewards/tag_count_reward/mean": 0.873046875, + "rewards/tag_count_reward/std": 0.2392423003911972, + "step": 1215 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1980.0, + "completions/mean_length": 797.548828125, + "completions/mean_terminated_length": 736.0512084960938, + "completions/min_length": 161.0, + "completions/min_terminated_length": 161.0, + "epoch": 0.4151233250832124, + "grad_norm": 2.610924482345581, + "kl": 6.3515625, + "learning_rate": 7.542718407357973e-07, + "loss": 0.4493, + "num_tokens": 702366438.0, + "reward": 1.7490234375, + "reward_std": 0.6174426078796387, + "rewards/accuracy_reward/mean": 0.06451612710952759, + "rewards/accuracy_reward/std": 0.2459181249141693, + "rewards/format_reward/mean": 0.7890625, + "rewards/format_reward/std": 0.4083731174468994, + "rewards/tag_count_reward/mean": 0.8974609375, + "rewards/tag_count_reward/std": 0.2228032499551773, + "step": 1216 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 823.921875, + "completions/mean_terminated_length": 784.4354858398438, + "completions/min_length": 176.0, + "completions/min_terminated_length": 176.0, + "epoch": 0.41546470939660324, + "grad_norm": 1.7523225545883179, + "kl": 6.328125, + "learning_rate": 7.537940056971192e-07, + "loss": 0.3804, + "num_tokens": 702862318.0, + "reward": 1.69921875, + "reward_std": 0.6397418975830078, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.767578125, + "rewards/format_reward/std": 0.42278963327407837, + "rewards/tag_count_reward/mean": 0.884765625, + "rewards/tag_count_reward/std": 0.22697781026363373, + "step": 1217 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2005.0, + "completions/mean_length": 755.7734375, + "completions/mean_terminated_length": 700.505126953125, + "completions/min_length": 76.0, + "completions/min_terminated_length": 76.0, + "epoch": 0.41580609370999405, + "grad_norm": 1.3652105331420898, + "kl": 6.296875, + "learning_rate": 7.533158814101242e-07, + "loss": 0.4089, + "num_tokens": 703334346.0, + "reward": 1.76318359375, + "reward_std": 0.6007625460624695, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.765625, + "rewards/format_reward/std": 0.42402184009552, + "rewards/tag_count_reward/mean": 0.89990234375, + "rewards/tag_count_reward/std": 0.21299511194229126, + "step": 1218 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 876.01171875, + "completions/mean_terminated_length": 797.8792114257812, + "completions/min_length": 17.0, + "completions/min_terminated_length": 17.0, + "epoch": 0.4161474780233848, + "grad_norm": 1.6425671577453613, + "kl": 8.3046875, + "learning_rate": 7.528374685534227e-07, + "loss": 0.5534, + "num_tokens": 703868752.0, + "reward": 1.6044921875, + "reward_std": 0.7019073367118835, + "rewards/accuracy_reward/mean": 0.05645161122083664, + "rewards/accuracy_reward/std": 0.23102475702762604, + "rewards/format_reward/mean": 0.69921875, + "rewards/format_reward/std": 0.45904624462127686, + "rewards/tag_count_reward/mean": 0.8505859375, + "rewards/tag_count_reward/std": 0.2596279978752136, + "step": 1219 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 864.052734375, + "completions/mean_terminated_length": 795.5598754882812, + "completions/min_length": 126.0, + "completions/min_terminated_length": 126.0, + "epoch": 0.4164888623367756, + "grad_norm": 1.81349778175354, + "kl": 6.1953125, + "learning_rate": 7.52358767806034e-07, + "loss": 0.4059, + "num_tokens": 704389803.0, + "reward": 1.75927734375, + "reward_std": 0.6504988670349121, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.77734375, + "rewards/format_reward/std": 0.41643625497817993, + "rewards/tag_count_reward/mean": 0.88623046875, + "rewards/tag_count_reward/std": 0.23118187487125397, + "step": 1220 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1935.0, + "completions/mean_length": 879.876953125, + "completions/mean_terminated_length": 807.1722412109375, + "completions/min_length": 204.0, + "completions/min_terminated_length": 204.0, + "epoch": 0.41683024665016644, + "grad_norm": 1.5401265621185303, + "kl": 6.890625, + "learning_rate": 7.518797798473865e-07, + "loss": 0.4578, + "num_tokens": 704917340.0, + "reward": 1.7490234375, + "reward_std": 0.6217037439346313, + "rewards/accuracy_reward/mean": 0.087890625, + "rewards/accuracy_reward/std": 0.2834126651287079, + "rewards/format_reward/mean": 0.7734375, + "rewards/format_reward/std": 0.4190165400505066, + "rewards/tag_count_reward/mean": 0.8876953125, + "rewards/tag_count_reward/std": 0.23110590875148773, + "step": 1221 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 832.552734375, + "completions/mean_terminated_length": 775.3844604492188, + "completions/min_length": 113.0, + "completions/min_terminated_length": 113.0, + "epoch": 0.41717163096355725, + "grad_norm": 2.129183292388916, + "kl": 4.94921875, + "learning_rate": 7.514005053573156e-07, + "loss": 0.3138, + "num_tokens": 705418391.0, + "reward": 1.8369140625, + "reward_std": 0.5654664039611816, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.822265625, + "rewards/format_reward/std": 0.3826628625392914, + "rewards/tag_count_reward/mean": 0.9189453125, + "rewards/tag_count_reward/std": 0.20277541875839233, + "step": 1222 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 780.529296875, + "completions/mean_terminated_length": 737.0000610351562, + "completions/min_length": 75.0, + "completions/min_terminated_length": 75.0, + "epoch": 0.417513015276948, + "grad_norm": 2.336930990219116, + "kl": 4.640625, + "learning_rate": 7.509209450160639e-07, + "loss": 0.3067, + "num_tokens": 705897702.0, + "reward": 1.80859375, + "reward_std": 0.5186939239501953, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.83203125, + "rewards/format_reward/std": 0.374204158782959, + "rewards/tag_count_reward/mean": 0.921875, + "rewards/tag_count_reward/std": 0.19597215950489044, + "step": 1223 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1942.0, + "completions/mean_length": 790.908203125, + "completions/mean_terminated_length": 745.103271484375, + "completions/min_length": 120.0, + "completions/min_terminated_length": 120.0, + "epoch": 0.4178543995903388, + "grad_norm": 3.638871908187866, + "kl": 3.94921875, + "learning_rate": 7.5044109950428e-07, + "loss": 0.3141, + "num_tokens": 706375447.0, + "reward": 1.8720703125, + "reward_std": 0.4504891335964203, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.8828125, + "rewards/format_reward/std": 0.32195815443992615, + "rewards/tag_count_reward/mean": 0.9423828125, + "rewards/tag_count_reward/std": 0.1758417934179306, + "step": 1224 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1884.0, + "completions/mean_length": 725.037109375, + "completions/mean_terminated_length": 687.8453369140625, + "completions/min_length": 170.0, + "completions/min_terminated_length": 170.0, + "epoch": 0.41819578390372963, + "grad_norm": 2.105980634689331, + "kl": 3.8203125, + "learning_rate": 7.499609695030163e-07, + "loss": 0.2773, + "num_tokens": 706821706.0, + "reward": 1.90087890625, + "reward_std": 0.4926224946975708, + "rewards/accuracy_reward/mean": 0.087890625, + "rewards/accuracy_reward/std": 0.2834126651287079, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.94384765625, + "rewards/tag_count_reward/std": 0.1674204021692276, + "step": 1225 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1997.0, + "completions/mean_length": 788.541015625, + "completions/mean_terminated_length": 745.2869262695312, + "completions/min_length": 205.0, + "completions/min_terminated_length": 205.0, + "epoch": 0.41853716821712045, + "grad_norm": 2.395641565322876, + "kl": 6.296875, + "learning_rate": 7.494805556937299e-07, + "loss": 0.4447, + "num_tokens": 707303775.0, + "reward": 1.806640625, + "reward_std": 0.5870789289474487, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.828125, + "rewards/format_reward/std": 0.3776407241821289, + "rewards/tag_count_reward/mean": 0.912109375, + "rewards/tag_count_reward/std": 0.20946265757083893, + "step": 1226 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1978.0, + "completions/mean_length": 764.029296875, + "completions/mean_terminated_length": 746.231689453125, + "completions/min_length": 149.0, + "completions/min_terminated_length": 149.0, + "epoch": 0.4188785525305112, + "grad_norm": 1.5705026388168335, + "kl": 6.0, + "learning_rate": 7.4899985875828e-07, + "loss": 0.3616, + "num_tokens": 707772318.0, + "reward": 1.841796875, + "reward_std": 0.5263835191726685, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.931640625, + "rewards/tag_count_reward/std": 0.18759170174598694, + "step": 1227 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1880.0, + "completions/mean_length": 753.322265625, + "completions/mean_terminated_length": 722.2500610351562, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "epoch": 0.419219936843902, + "grad_norm": 1.6948952674865723, + "kl": 5.75, + "learning_rate": 7.485188793789284e-07, + "loss": 0.3697, + "num_tokens": 708240227.0, + "reward": 1.84130859375, + "reward_std": 0.5240170955657959, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.93115234375, + "rewards/tag_count_reward/std": 0.1837882399559021, + "step": 1228 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2005.0, + "completions/mean_length": 798.45703125, + "completions/mean_terminated_length": 731.6090087890625, + "completions/min_length": 139.0, + "completions/min_terminated_length": 139.0, + "epoch": 0.41956132115729283, + "grad_norm": 7.500605583190918, + "kl": 10.9375, + "learning_rate": 7.480376182383371e-07, + "loss": 0.6152, + "num_tokens": 708728573.0, + "reward": 1.78076171875, + "reward_std": 0.658714771270752, + "rewards/accuracy_reward/mean": 0.103515625, + "rewards/accuracy_reward/std": 0.30492907762527466, + "rewards/format_reward/mean": 0.78515625, + "rewards/format_reward/std": 0.4111155867576599, + "rewards/tag_count_reward/mean": 0.89208984375, + "rewards/tag_count_reward/std": 0.22491775453090668, + "step": 1229 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2033.0, + "completions/mean_length": 728.208984375, + "completions/mean_terminated_length": 688.376220703125, + "completions/min_length": 213.0, + "completions/min_terminated_length": 213.0, + "epoch": 0.41990270547068365, + "grad_norm": 6.722501754760742, + "kl": 8.640625, + "learning_rate": 7.47556076019568e-07, + "loss": 0.4902, + "num_tokens": 709182520.0, + "reward": 1.830078125, + "reward_std": 0.5410079956054688, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.830078125, + "rewards/format_reward/std": 0.3759314715862274, + "rewards/tag_count_reward/mean": 0.92578125, + "rewards/tag_count_reward/std": 0.1937359869480133, + "step": 1230 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2009.0, + "completions/mean_length": 752.740234375, + "completions/mean_terminated_length": 708.256591796875, + "completions/min_length": 167.0, + "completions/min_terminated_length": 167.0, + "epoch": 0.4202440897840744, + "grad_norm": 5.435886859893799, + "kl": 9.375, + "learning_rate": 7.470742534060827e-07, + "loss": 0.5561, + "num_tokens": 709643955.0, + "reward": 1.79248046875, + "reward_std": 0.6186740398406982, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.791015625, + "rewards/format_reward/std": 0.40698084235191345, + "rewards/tag_count_reward/mean": 0.90966796875, + "rewards/tag_count_reward/std": 0.20516635477542877, + "step": 1231 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1975.0, + "completions/mean_length": 756.39453125, + "completions/mean_terminated_length": 712.036376953125, + "completions/min_length": 200.0, + "completions/min_terminated_length": 200.0, + "epoch": 0.4205854740974652, + "grad_norm": 6.821199893951416, + "kl": 8.703125, + "learning_rate": 7.465921510817401e-07, + "loss": 0.5082, + "num_tokens": 710121453.0, + "reward": 1.8095703125, + "reward_std": 0.5002506971359253, + "rewards/accuracy_reward/mean": 0.06653226166963577, + "rewards/accuracy_reward/std": 0.2494617998600006, + "rewards/format_reward/mean": 0.828125, + "rewards/format_reward/std": 0.3776407241821289, + "rewards/tag_count_reward/mean": 0.9169921875, + "rewards/tag_count_reward/std": 0.20379072427749634, + "step": 1232 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2016.0, + "completions/mean_length": 776.669921875, + "completions/mean_terminated_length": 724.9898071289062, + "completions/min_length": 208.0, + "completions/min_terminated_length": 208.0, + "epoch": 0.42092685841085603, + "grad_norm": 7.6722822189331055, + "kl": 10.03125, + "learning_rate": 7.461097697307962e-07, + "loss": 0.573, + "num_tokens": 710595492.0, + "reward": 1.73291015625, + "reward_std": 0.6036777496337891, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.78515625, + "rewards/format_reward/std": 0.4111155867576599, + "rewards/tag_count_reward/mean": 0.89697265625, + "rewards/tag_count_reward/std": 0.22064577043056488, + "step": 1233 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2033.0, + "completions/mean_length": 759.509765625, + "completions/mean_terminated_length": 707.132080078125, + "completions/min_length": 169.0, + "completions/min_terminated_length": 169.0, + "epoch": 0.42126824272424684, + "grad_norm": 1.8243650197982788, + "kl": 7.5546875, + "learning_rate": 7.456271100379031e-07, + "loss": 0.5141, + "num_tokens": 711057993.0, + "reward": 1.79345703125, + "reward_std": 0.573222279548645, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.830078125, + "rewards/format_reward/std": 0.3759314715862274, + "rewards/tag_count_reward/mean": 0.91650390625, + "rewards/tag_count_reward/std": 0.20922017097473145, + "step": 1234 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1812.0, + "completions/mean_length": 745.484375, + "completions/mean_terminated_length": 687.0040283203125, + "completions/min_length": 144.0, + "completions/min_terminated_length": 144.0, + "epoch": 0.4216096270376376, + "grad_norm": 1.6971056461334229, + "kl": 7.0234375, + "learning_rate": 7.451441726881082e-07, + "loss": 0.4601, + "num_tokens": 711523889.0, + "reward": 1.77783203125, + "reward_std": 0.5547708868980408, + "rewards/accuracy_reward/mean": 0.033203125, + "rewards/accuracy_reward/std": 0.17934183776378632, + "rewards/format_reward/mean": 0.830078125, + "rewards/format_reward/std": 0.3759314715862274, + "rewards/tag_count_reward/mean": 0.91455078125, + "rewards/tag_count_reward/std": 0.21192020177841187, + "step": 1235 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 818.845703125, + "completions/mean_terminated_length": 766.2749633789062, + "completions/min_length": 189.0, + "completions/min_terminated_length": 189.0, + "epoch": 0.4219510113510284, + "grad_norm": 1.6898390054702759, + "kl": 6.1875, + "learning_rate": 7.446609583668522e-07, + "loss": 0.4155, + "num_tokens": 712037026.0, + "reward": 1.82666015625, + "reward_std": 0.6324109435081482, + "rewards/accuracy_reward/mean": 0.1171875, + "rewards/accuracy_reward/std": 0.32195815443992615, + "rewards/format_reward/mean": 0.802734375, + "rewards/format_reward/std": 0.3983237147331238, + "rewards/tag_count_reward/mean": 0.90673828125, + "rewards/tag_count_reward/std": 0.2115049660205841, + "step": 1236 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1965.0, + "completions/mean_length": 725.25390625, + "completions/mean_terminated_length": 688.0682373046875, + "completions/min_length": 150.0, + "completions/min_terminated_length": 150.0, + "epoch": 0.42229239566441923, + "grad_norm": 1.8300591707229614, + "kl": 5.79296875, + "learning_rate": 7.441774677599699e-07, + "loss": 0.3901, + "num_tokens": 712485332.0, + "reward": 1.84521484375, + "reward_std": 0.5363088846206665, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.93115234375, + "rewards/tag_count_reward/std": 0.18577399849891663, + "step": 1237 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1907.0, + "completions/mean_length": 766.623046875, + "completions/mean_terminated_length": 700.843994140625, + "completions/min_length": 158.0, + "completions/min_terminated_length": 158.0, + "epoch": 0.42263377997781004, + "grad_norm": 2.1360459327697754, + "kl": 6.0234375, + "learning_rate": 7.436937015536876e-07, + "loss": 0.3957, + "num_tokens": 712956419.0, + "reward": 1.7958984375, + "reward_std": 0.573586106300354, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.818359375, + "rewards/format_reward/std": 0.38592514395713806, + "rewards/tag_count_reward/mean": 0.9169921875, + "rewards/tag_count_reward/std": 0.20617742836475372, + "step": 1238 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 746.923828125, + "completions/mean_terminated_length": 707.6558837890625, + "completions/min_length": 34.0, + "completions/min_terminated_length": 34.0, + "epoch": 0.4229751642912008, + "grad_norm": 3.2511074542999268, + "kl": 5.9609375, + "learning_rate": 7.432096604346231e-07, + "loss": 0.4301, + "num_tokens": 713414972.0, + "reward": 1.82470703125, + "reward_std": 0.5387951135635376, + "rewards/accuracy_reward/mean": 0.05443548411130905, + "rewards/accuracy_reward/std": 0.227104052901268, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.93212890625, + "rewards/tag_count_reward/std": 0.18481481075286865, + "step": 1239 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1987.0, + "completions/mean_length": 769.263671875, + "completions/mean_terminated_length": 738.5740356445312, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "epoch": 0.4233165486045916, + "grad_norm": 1.5109126567840576, + "kl": 4.640625, + "learning_rate": 7.427253450897844e-07, + "loss": 0.2805, + "num_tokens": 713888643.0, + "reward": 1.78271484375, + "reward_std": 0.5137543082237244, + "rewards/accuracy_reward/mean": 0.03125, + "rewards/accuracy_reward/std": 0.17416280508041382, + "rewards/format_reward/mean": 0.82421875, + "rewards/format_reward/std": 0.3810062110424042, + "rewards/tag_count_reward/mean": 0.92724609375, + "rewards/tag_count_reward/std": 0.18625685572624207, + "step": 1240 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1735.0, + "completions/mean_length": 800.2421875, + "completions/mean_terminated_length": 749.5203247070312, + "completions/min_length": 224.0, + "completions/min_terminated_length": 224.0, + "epoch": 0.4236579329179824, + "grad_norm": 1.2704123258590698, + "kl": 6.0859375, + "learning_rate": 7.422407562065678e-07, + "loss": 0.3967, + "num_tokens": 714374799.0, + "reward": 1.85009765625, + "reward_std": 0.5252021551132202, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.92822265625, + "rewards/tag_count_reward/std": 0.19307827949523926, + "step": 1241 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2003.0, + "completions/mean_length": 819.794921875, + "completions/mean_terminated_length": 767.2648315429688, + "completions/min_length": 206.0, + "completions/min_terminated_length": 206.0, + "epoch": 0.42399931723137324, + "grad_norm": 2.8821945190429688, + "kl": 8.484375, + "learning_rate": 7.417558944727593e-07, + "loss": 0.5146, + "num_tokens": 714881830.0, + "reward": 1.75341796875, + "reward_std": 0.602732241153717, + "rewards/accuracy_reward/mean": 0.07258064299821854, + "rewards/accuracy_reward/std": 0.25970885157585144, + "rewards/format_reward/mean": 0.779296875, + "rewards/format_reward/std": 0.4151262938976288, + "rewards/tag_count_reward/mean": 0.90380859375, + "rewards/tag_count_reward/std": 0.20725619792938232, + "step": 1242 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 772.87890625, + "completions/mean_terminated_length": 721.044677734375, + "completions/min_length": 188.0, + "completions/min_terminated_length": 188.0, + "epoch": 0.424340701544764, + "grad_norm": 5.812600612640381, + "kl": 10.8125, + "learning_rate": 7.412707605765313e-07, + "loss": 0.6363, + "num_tokens": 715354312.0, + "reward": 1.69921875, + "reward_std": 0.6376601457595825, + "rewards/accuracy_reward/mean": 0.037109375, + "rewards/accuracy_reward/std": 0.18921469151973724, + "rewards/format_reward/mean": 0.76953125, + "rewards/format_reward/std": 0.42154473066329956, + "rewards/tag_count_reward/mean": 0.892578125, + "rewards/tag_count_reward/std": 0.22812001407146454, + "step": 1243 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2013.0, + "completions/mean_length": 793.798828125, + "completions/mean_terminated_length": 763.6980590820312, + "completions/min_length": 217.0, + "completions/min_terminated_length": 217.0, + "epoch": 0.4246820858581548, + "grad_norm": 3.165933609008789, + "kl": 7.3671875, + "learning_rate": 7.407853552064425e-07, + "loss": 0.4275, + "num_tokens": 715839825.0, + "reward": 1.83447265625, + "reward_std": 0.575330376625061, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.826171875, + "rewards/format_reward/std": 0.3793322443962097, + "rewards/tag_count_reward/mean": 0.91650390625, + "rewards/tag_count_reward/std": 0.20025932788848877, + "step": 1244 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 780.966796875, + "completions/mean_terminated_length": 747.9579467773438, + "completions/min_length": 205.0, + "completions/min_terminated_length": 205.0, + "epoch": 0.4250234701715456, + "grad_norm": 1.9695323705673218, + "kl": 7.546875, + "learning_rate": 7.40299679051437e-07, + "loss": 0.46, + "num_tokens": 716316496.0, + "reward": 1.76806640625, + "reward_std": 0.5813748836517334, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.802734375, + "rewards/format_reward/std": 0.3983237147331238, + "rewards/tag_count_reward/mean": 0.91455078125, + "rewards/tag_count_reward/std": 0.19820177555084229, + "step": 1245 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2000.0, + "completions/mean_length": 827.5390625, + "completions/mean_terminated_length": 759.5958862304688, + "completions/min_length": 155.0, + "completions/min_terminated_length": 155.0, + "epoch": 0.42536485448493644, + "grad_norm": 4.001309394836426, + "kl": 9.09375, + "learning_rate": 7.398137328008435e-07, + "loss": 0.521, + "num_tokens": 716828212.0, + "reward": 1.712890625, + "reward_std": 0.6477146744728088, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.76171875, + "rewards/format_reward/std": 0.42644867300987244, + "rewards/tag_count_reward/mean": 0.890625, + "rewards/tag_count_reward/std": 0.22879758477210999, + "step": 1246 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1951.0, + "completions/mean_length": 797.966796875, + "completions/mean_terminated_length": 749.7910766601562, + "completions/min_length": 219.0, + "completions/min_terminated_length": 219.0, + "epoch": 0.4257062387983272, + "grad_norm": 1.3594669103622437, + "kl": 6.45703125, + "learning_rate": 7.393275171443737e-07, + "loss": 0.3914, + "num_tokens": 717316627.0, + "reward": 1.77197265625, + "reward_std": 0.5782288312911987, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.77734375, + "rewards/format_reward/std": 0.41643625497817993, + "rewards/tag_count_reward/mean": 0.90283203125, + "rewards/tag_count_reward/std": 0.21263141930103302, + "step": 1247 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1960.0, + "completions/mean_length": 825.515625, + "completions/mean_terminated_length": 762.759765625, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "epoch": 0.426047623111718, + "grad_norm": 1.1935927867889404, + "kl": 6.5703125, + "learning_rate": 7.388410327721218e-07, + "loss": 0.4069, + "num_tokens": 717821659.0, + "reward": 1.7099609375, + "reward_std": 0.5673522353172302, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.775390625, + "rewards/format_reward/std": 0.41773295402526855, + "rewards/tag_count_reward/mean": 0.8935546875, + "rewards/tag_count_reward/std": 0.23229363560676575, + "step": 1248 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1921.0, + "completions/mean_length": 816.89453125, + "completions/mean_terminated_length": 772.0364379882812, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "epoch": 0.4263890074251088, + "grad_norm": 1.9006518125534058, + "kl": 6.1875, + "learning_rate": 7.383542803745632e-07, + "loss": 0.3989, + "num_tokens": 718311525.0, + "reward": 1.759765625, + "reward_std": 0.6409410834312439, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.791015625, + "rewards/format_reward/std": 0.40698084235191345, + "rewards/tag_count_reward/mean": 0.896484375, + "rewards/tag_count_reward/std": 0.22670821845531464, + "step": 1249 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2015.0, + "completions/mean_length": 844.4765625, + "completions/mean_terminated_length": 785.286865234375, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "epoch": 0.42673039173849964, + "grad_norm": 0.9973497986793518, + "kl": 7.6171875, + "learning_rate": 7.378672606425542e-07, + "loss": 0.4726, + "num_tokens": 718816281.0, + "reward": 1.6767578125, + "reward_std": 0.683112382888794, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.740234375, + "rewards/format_reward/std": 0.4389347732067108, + "rewards/tag_count_reward/mean": 0.8642578125, + "rewards/tag_count_reward/std": 0.25485852360725403, + "step": 1250 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1970.0, + "completions/mean_length": 862.634765625, + "completions/mean_terminated_length": 796.6453857421875, + "completions/min_length": 240.0, + "completions/min_terminated_length": 240.0, + "epoch": 0.4270717760518904, + "grad_norm": 2.722085475921631, + "kl": 7.15625, + "learning_rate": 7.373799742673301e-07, + "loss": 0.4986, + "num_tokens": 719329614.0, + "reward": 1.732421875, + "reward_std": 0.6683371663093567, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.7578125, + "rewards/format_reward/std": 0.42882615327835083, + "rewards/tag_count_reward/mean": 0.890625, + "rewards/tag_count_reward/std": 0.22118698060512543, + "step": 1251 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2007.0, + "completions/mean_length": 828.90234375, + "completions/mean_terminated_length": 763.68310546875, + "completions/min_length": 31.0, + "completions/min_terminated_length": 31.0, + "epoch": 0.4274131603652812, + "grad_norm": 2.0579171180725098, + "kl": 6.4375, + "learning_rate": 7.36892421940505e-07, + "loss": 0.4052, + "num_tokens": 719825644.0, + "reward": 1.744140625, + "reward_std": 0.6250649690628052, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.791015625, + "rewards/format_reward/std": 0.40698084235191345, + "rewards/tag_count_reward/mean": 0.8984375, + "rewards/tag_count_reward/std": 0.22434400022029877, + "step": 1252 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1969.0, + "completions/mean_length": 863.185546875, + "completions/mean_terminated_length": 797.226806640625, + "completions/min_length": 164.0, + "completions/min_terminated_length": 164.0, + "epoch": 0.427754544678672, + "grad_norm": 2.6889405250549316, + "kl": 6.8984375, + "learning_rate": 7.364046043540699e-07, + "loss": 0.4256, + "num_tokens": 720349179.0, + "reward": 1.7109375, + "reward_std": 0.6374015808105469, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.76171875, + "rewards/format_reward/std": 0.42644867300987244, + "rewards/tag_count_reward/mean": 0.88671875, + "rewards/tag_count_reward/std": 0.23845018446445465, + "step": 1253 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.060546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2000.0, + "completions/mean_length": 849.505859375, + "completions/mean_terminated_length": 772.2640380859375, + "completions/min_length": 69.0, + "completions/min_terminated_length": 69.0, + "epoch": 0.42809592899206284, + "grad_norm": 1.7696506977081299, + "kl": 7.5, + "learning_rate": 7.359165222003927e-07, + "loss": 0.4706, + "num_tokens": 720863150.0, + "reward": 1.697265625, + "reward_std": 0.6631327867507935, + "rewards/accuracy_reward/mean": 0.06451612710952759, + "rewards/accuracy_reward/std": 0.2459181249141693, + "rewards/format_reward/mean": 0.755859375, + "rewards/format_reward/std": 0.42999663949012756, + "rewards/tag_count_reward/mean": 0.87890625, + "rewards/tag_count_reward/std": 0.24125482141971588, + "step": 1254 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2000.0, + "completions/mean_length": 802.404296875, + "completions/mean_terminated_length": 754.3995971679688, + "completions/min_length": 151.0, + "completions/min_terminated_length": 151.0, + "epoch": 0.4284373133054536, + "grad_norm": 2.0506370067596436, + "kl": 5.53125, + "learning_rate": 7.354281761722168e-07, + "loss": 0.3899, + "num_tokens": 721352189.0, + "reward": 1.81494140625, + "reward_std": 0.5642163753509521, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.818359375, + "rewards/format_reward/std": 0.38592514395713806, + "rewards/tag_count_reward/mean": 0.92041015625, + "rewards/tag_count_reward/std": 0.19817768037319183, + "step": 1255 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1989.0, + "completions/mean_length": 787.2421875, + "completions/mean_terminated_length": 730.63671875, + "completions/min_length": 178.0, + "completions/min_terminated_length": 178.0, + "epoch": 0.4287786976188444, + "grad_norm": 2.414344072341919, + "kl": 6.5546875, + "learning_rate": 7.349395669626601e-07, + "loss": 0.4446, + "num_tokens": 721829449.0, + "reward": 1.8193359375, + "reward_std": 0.6030614376068115, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.8203125, + "rewards/format_reward/std": 0.38430243730545044, + "rewards/tag_count_reward/mean": 0.9150390625, + "rewards/tag_count_reward/std": 0.20950597524642944, + "step": 1256 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1851.0, + "completions/mean_length": 751.89453125, + "completions/mean_terminated_length": 715.4578247070312, + "completions/min_length": 38.0, + "completions/min_terminated_length": 38.0, + "epoch": 0.4291200819322352, + "grad_norm": 0.9160401225090027, + "kl": 5.8359375, + "learning_rate": 7.344506952652141e-07, + "loss": 0.3673, + "num_tokens": 722288739.0, + "reward": 1.8515625, + "reward_std": 0.4972970485687256, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.9296875, + "rewards/tag_count_reward/std": 0.1939331740140915, + "step": 1257 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 796.3515625, + "completions/mean_terminated_length": 755.9757690429688, + "completions/min_length": 249.0, + "completions/min_terminated_length": 249.0, + "epoch": 0.42946146624562603, + "grad_norm": 1.0107033252716064, + "kl": 5.984375, + "learning_rate": 7.339615617737427e-07, + "loss": 0.3545, + "num_tokens": 722772663.0, + "reward": 1.82080078125, + "reward_std": 0.5160529613494873, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.93408203125, + "rewards/tag_count_reward/std": 0.18617989122867584, + "step": 1258 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 763.390625, + "completions/mean_terminated_length": 708.4481201171875, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "epoch": 0.4298028505590168, + "grad_norm": 1.9055343866348267, + "kl": 9.0390625, + "learning_rate": 7.334721671824814e-07, + "loss": 0.5632, + "num_tokens": 723239583.0, + "reward": 1.76806640625, + "reward_std": 0.5623615384101868, + "rewards/accuracy_reward/mean": 0.025390625, + "rewards/accuracy_reward/std": 0.15746226906776428, + "rewards/format_reward/mean": 0.822265625, + "rewards/format_reward/std": 0.3826628625392914, + "rewards/tag_count_reward/mean": 0.92041015625, + "rewards/tag_count_reward/std": 0.20063117146492004, + "step": 1259 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2007.0, + "completions/mean_length": 709.197265625, + "completions/mean_terminated_length": 682.5278930664062, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "epoch": 0.4301442348724076, + "grad_norm": 1.7391200065612793, + "kl": 6.6015625, + "learning_rate": 7.329825121860363e-07, + "loss": 0.378, + "num_tokens": 723684116.0, + "reward": 1.8203125, + "reward_std": 0.5584942102432251, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.833984375, + "rewards/format_reward/std": 0.3724585771560669, + "rewards/tag_count_reward/mean": 0.923828125, + "rewards/tag_count_reward/std": 0.1948670893907547, + "step": 1260 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1864.0, + "completions/mean_length": 751.787109375, + "completions/mean_terminated_length": 720.6780395507812, + "completions/min_length": 232.0, + "completions/min_terminated_length": 232.0, + "epoch": 0.4304856191857984, + "grad_norm": 1.2627638578414917, + "kl": 5.83984375, + "learning_rate": 7.32492597479383e-07, + "loss": 0.3666, + "num_tokens": 724143479.0, + "reward": 1.869140625, + "reward_std": 0.44199299812316895, + "rewards/accuracy_reward/mean": 0.05443548411130905, + "rewards/accuracy_reward/std": 0.227104052901268, + "rewards/format_reward/mean": 0.880859375, + "rewards/format_reward/std": 0.32427072525024414, + "rewards/tag_count_reward/mean": 0.935546875, + "rewards/tag_count_reward/std": 0.18036192655563354, + "step": 1261 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1985.0, + "completions/mean_length": 743.861328125, + "completions/mean_terminated_length": 717.8825073242188, + "completions/min_length": 174.0, + "completions/min_terminated_length": 174.0, + "epoch": 0.43082700349918923, + "grad_norm": 1.1336578130722046, + "kl": 4.88671875, + "learning_rate": 7.32002423757866e-07, + "loss": 0.3102, + "num_tokens": 724602064.0, + "reward": 1.8837890625, + "reward_std": 0.4746038317680359, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.875, + "rewards/format_reward/std": 0.3310423493385315, + "rewards/tag_count_reward/mean": 0.9404296875, + "rewards/tag_count_reward/std": 0.17237325012683868, + "step": 1262 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 749.705078125, + "completions/mean_terminated_length": 721.1995849609375, + "completions/min_length": 72.0, + "completions/min_terminated_length": 72.0, + "epoch": 0.43116838781258, + "grad_norm": 2.871920347213745, + "kl": 6.328125, + "learning_rate": 7.315119917171972e-07, + "loss": 0.4186, + "num_tokens": 725058857.0, + "reward": 1.84521484375, + "reward_std": 0.4899643063545227, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.93505859375, + "rewards/tag_count_reward/std": 0.1784812957048416, + "step": 1263 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2033.0, + "completions/mean_length": 794.529296875, + "completions/mean_terminated_length": 746.2210693359375, + "completions/min_length": 167.0, + "completions/min_terminated_length": 167.0, + "epoch": 0.4315097721259708, + "grad_norm": 1.7654612064361572, + "kl": 7.21875, + "learning_rate": 7.310213020534549e-07, + "loss": 0.4814, + "num_tokens": 725552280.0, + "reward": 1.75830078125, + "reward_std": 0.5049154162406921, + "rewards/accuracy_reward/mean": 0.0078125, + "rewards/accuracy_reward/std": 0.08812850713729858, + "rewards/format_reward/mean": 0.83203125, + "rewards/format_reward/std": 0.374204158782959, + "rewards/tag_count_reward/mean": 0.91845703125, + "rewards/tag_count_reward/std": 0.2022770643234253, + "step": 1264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1970.0, + "completions/mean_length": 740.4453125, + "completions/mean_terminated_length": 687.2926635742188, + "completions/min_length": 127.0, + "completions/min_terminated_length": 127.0, + "epoch": 0.4318511564393616, + "grad_norm": 5.806798934936523, + "kl": 9.05859375, + "learning_rate": 7.30530355463084e-07, + "loss": 0.5213, + "num_tokens": 726011420.0, + "reward": 1.80078125, + "reward_std": 0.6177265644073486, + "rewards/accuracy_reward/mean": 0.109375, + "rewards/accuracy_reward/std": 0.31241437792778015, + "rewards/format_reward/mean": 0.791015625, + "rewards/format_reward/std": 0.40698084235191345, + "rewards/tag_count_reward/mean": 0.900390625, + "rewards/tag_count_reward/std": 0.22138561308383942, + "step": 1265 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 758.076171875, + "completions/mean_terminated_length": 724.470947265625, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "epoch": 0.43219254075275243, + "grad_norm": 3.785024642944336, + "kl": 8.96875, + "learning_rate": 7.300391526428928e-07, + "loss": 0.5283, + "num_tokens": 726481219.0, + "reward": 1.74169921875, + "reward_std": 0.6067723035812378, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.78515625, + "rewards/format_reward/std": 0.4111155867576599, + "rewards/tag_count_reward/mean": 0.90966796875, + "rewards/tag_count_reward/std": 0.2009500116109848, + "step": 1266 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2000.0, + "completions/mean_length": 779.455078125, + "completions/mean_terminated_length": 725.1996459960938, + "completions/min_length": 161.0, + "completions/min_terminated_length": 161.0, + "epoch": 0.4325339250661432, + "grad_norm": 3.3844306468963623, + "kl": 8.3828125, + "learning_rate": 7.295476942900539e-07, + "loss": 0.4816, + "num_tokens": 726966204.0, + "reward": 1.7646484375, + "reward_std": 0.626046895980835, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.76953125, + "rewards/format_reward/std": 0.42154473066329956, + "rewards/tag_count_reward/mean": 0.8974609375, + "rewards/tag_count_reward/std": 0.21096043288707733, + "step": 1267 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.056640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2018.0, + "completions/mean_length": 788.740234375, + "completions/mean_terminated_length": 713.1325073242188, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.432875309379534, + "grad_norm": 3.069222927093506, + "kl": 9.65625, + "learning_rate": 7.290559811021029e-07, + "loss": 0.593, + "num_tokens": 727448071.0, + "reward": 1.783203125, + "reward_std": 0.6747581958770752, + "rewards/accuracy_reward/mean": 0.126953125, + "rewards/accuracy_reward/std": 0.33324605226516724, + "rewards/format_reward/mean": 0.775390625, + "rewards/format_reward/std": 0.41773295402526855, + "rewards/tag_count_reward/mean": 0.880859375, + "rewards/tag_count_reward/std": 0.23609022796154022, + "step": 1268 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2030.0, + "completions/mean_length": 791.298828125, + "completions/mean_terminated_length": 742.8660888671875, + "completions/min_length": 113.0, + "completions/min_terminated_length": 113.0, + "epoch": 0.4332166936929248, + "grad_norm": 1.4948716163635254, + "kl": 7.2265625, + "learning_rate": 7.285640137769363e-07, + "loss": 0.4889, + "num_tokens": 727925920.0, + "reward": 1.78662109375, + "reward_std": 0.6075412034988403, + "rewards/accuracy_reward/mean": 0.087890625, + "rewards/accuracy_reward/std": 0.2834126651287079, + "rewards/format_reward/mean": 0.80078125, + "rewards/format_reward/std": 0.39980348944664, + "rewards/tag_count_reward/mean": 0.89794921875, + "rewards/tag_count_reward/std": 0.22924767434597015, + "step": 1269 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1998.0, + "completions/mean_length": 837.33984375, + "completions/mean_terminated_length": 795.7616577148438, + "completions/min_length": 181.0, + "completions/min_terminated_length": 181.0, + "epoch": 0.43355807800631563, + "grad_norm": 1.5656808614730835, + "kl": 6.9765625, + "learning_rate": 7.280717930128119e-07, + "loss": 0.451, + "num_tokens": 728426094.0, + "reward": 1.7724609375, + "reward_std": 0.6285260915756226, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.779296875, + "rewards/format_reward/std": 0.4151262938976288, + "rewards/tag_count_reward/mean": 0.8955078125, + "rewards/tag_count_reward/std": 0.20588061213493347, + "step": 1270 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2008.0, + "completions/mean_length": 772.10546875, + "completions/mean_terminated_length": 706.6078491210938, + "completions/min_length": 186.0, + "completions/min_terminated_length": 186.0, + "epoch": 0.4338994623197064, + "grad_norm": 3.605860710144043, + "kl": 6.1484375, + "learning_rate": 7.275793195083474e-07, + "loss": 0.4371, + "num_tokens": 728898676.0, + "reward": 1.75, + "reward_std": 0.604926347732544, + "rewards/accuracy_reward/mean": 0.05645161122083664, + "rewards/accuracy_reward/std": 0.23102475702762604, + "rewards/format_reward/mean": 0.791015625, + "rewards/format_reward/std": 0.40698084235191345, + "rewards/tag_count_reward/mean": 0.904296875, + "rewards/tag_count_reward/std": 0.2175542116165161, + "step": 1271 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1966.0, + "completions/mean_length": 777.689453125, + "completions/mean_terminated_length": 754.960205078125, + "completions/min_length": 144.0, + "completions/min_terminated_length": 144.0, + "epoch": 0.4342408466330972, + "grad_norm": 2.478461265563965, + "kl": 3.43359375, + "learning_rate": 7.270865939625183e-07, + "loss": 0.2044, + "num_tokens": 729375685.0, + "reward": 1.83935546875, + "reward_std": 0.4839392304420471, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.93115234375, + "rewards/tag_count_reward/std": 0.175620898604393, + "step": 1272 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1992.0, + "completions/mean_length": 754.076171875, + "completions/mean_terminated_length": 728.3008422851562, + "completions/min_length": 10.0, + "completions/min_terminated_length": 10.0, + "epoch": 0.434582230946488, + "grad_norm": 2.1076242923736572, + "kl": 4.44921875, + "learning_rate": 7.265936170746588e-07, + "loss": 0.2837, + "num_tokens": 729833676.0, + "reward": 1.83447265625, + "reward_std": 0.5441898107528687, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.83203125, + "rewards/format_reward/std": 0.374204158782959, + "rewards/tag_count_reward/mean": 0.92431640625, + "rewards/tag_count_reward/std": 0.18640044331550598, + "step": 1273 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2021.0, + "completions/mean_length": 845.30078125, + "completions/mean_terminated_length": 778.346435546875, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "epoch": 0.4349236152598788, + "grad_norm": 1.2890641689300537, + "kl": 7.0234375, + "learning_rate": 7.261003895444593e-07, + "loss": 0.4465, + "num_tokens": 730338278.0, + "reward": 1.720703125, + "reward_std": 0.6220524907112122, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.763671875, + "rewards/format_reward/std": 0.42524150013923645, + "rewards/tag_count_reward/mean": 0.890625, + "rewards/tag_count_reward/std": 0.22229015827178955, + "step": 1274 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 857.912109375, + "completions/mean_terminated_length": 801.9365844726562, + "completions/min_length": 146.0, + "completions/min_terminated_length": 146.0, + "epoch": 0.4352649995732696, + "grad_norm": 2.3086037635803223, + "kl": 7.453125, + "learning_rate": 7.256069120719661e-07, + "loss": 0.4659, + "num_tokens": 730845465.0, + "reward": 1.744140625, + "reward_std": 0.6320393085479736, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.78125, + "rewards/format_reward/std": 0.41380295157432556, + "rewards/tag_count_reward/mean": 0.88671875, + "rewards/tag_count_reward/std": 0.23431077599525452, + "step": 1275 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2033.0, + "completions/mean_length": 780.814453125, + "completions/mean_terminated_length": 742.5693969726562, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "epoch": 0.4356063838866604, + "grad_norm": 1.7257732152938843, + "kl": 6.84375, + "learning_rate": 7.2511318535758e-07, + "loss": 0.408, + "num_tokens": 731321274.0, + "reward": 1.724609375, + "reward_std": 0.6101690530776978, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.767578125, + "rewards/format_reward/std": 0.42278963327407837, + "rewards/tag_count_reward/mean": 0.892578125, + "rewards/tag_count_reward/std": 0.21993058919906616, + "step": 1276 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2015.0, + "completions/mean_length": 795.44921875, + "completions/mean_terminated_length": 747.1764526367188, + "completions/min_length": 165.0, + "completions/min_terminated_length": 165.0, + "epoch": 0.4359477682000512, + "grad_norm": 2.6494693756103516, + "kl": 7.5625, + "learning_rate": 7.246192101020559e-07, + "loss": 0.4435, + "num_tokens": 731797984.0, + "reward": 1.720703125, + "reward_std": 0.604543149471283, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.767578125, + "rewards/format_reward/std": 0.42278963327407837, + "rewards/tag_count_reward/mean": 0.884765625, + "rewards/tag_count_reward/std": 0.23387810587882996, + "step": 1277 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1912.0, + "completions/mean_length": 778.177734375, + "completions/mean_terminated_length": 737.2156982421875, + "completions/min_length": 134.0, + "completions/min_terminated_length": 134.0, + "epoch": 0.436289152513442, + "grad_norm": 1.5474677085876465, + "kl": 5.46875, + "learning_rate": 7.241249870065014e-07, + "loss": 0.3395, + "num_tokens": 732274187.0, + "reward": 1.7666015625, + "reward_std": 0.5756736397743225, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.806640625, + "rewards/format_reward/std": 0.39531853795051575, + "rewards/tag_count_reward/mean": 0.9111328125, + "rewards/tag_count_reward/std": 0.20129980146884918, + "step": 1278 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 837.470703125, + "completions/mean_terminated_length": 762.1265869140625, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "epoch": 0.4366305368268328, + "grad_norm": 1.7922933101654053, + "kl": 7.546875, + "learning_rate": 7.236305167723758e-07, + "loss": 0.4332, + "num_tokens": 732782540.0, + "reward": 1.7109375, + "reward_std": 0.654686450958252, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.7578125, + "rewards/format_reward/std": 0.42882615327835083, + "rewards/tag_count_reward/mean": 0.873046875, + "rewards/tag_count_reward/std": 0.24679172039031982, + "step": 1279 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1996.0, + "completions/mean_length": 832.134765625, + "completions/mean_terminated_length": 782.7093505859375, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "epoch": 0.4369719211402236, + "grad_norm": 0.7395869493484497, + "kl": 7.3515625, + "learning_rate": 7.231358001014891e-07, + "loss": 0.4439, + "num_tokens": 733279329.0, + "reward": 1.71826171875, + "reward_std": 0.5961915850639343, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.775390625, + "rewards/format_reward/std": 0.41773295402526855, + "rewards/tag_count_reward/mean": 0.89404296875, + "rewards/tag_count_reward/std": 0.21869266033172607, + "step": 1280 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1906.0, + "completions/mean_length": 848.68359375, + "completions/mean_terminated_length": 789.7008056640625, + "completions/min_length": 122.0, + "completions/min_terminated_length": 122.0, + "epoch": 0.4373133054536144, + "grad_norm": 1.4757400751113892, + "kl": 6.90625, + "learning_rate": 7.22640837696001e-07, + "loss": 0.4081, + "num_tokens": 733801391.0, + "reward": 1.69677734375, + "reward_std": 0.6439635753631592, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.751953125, + "rewards/format_reward/std": 0.4323015511035919, + "rewards/tag_count_reward/mean": 0.88427734375, + "rewards/tag_count_reward/std": 0.22807340323925018, + "step": 1281 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1975.0, + "completions/mean_length": 739.98046875, + "completions/mean_terminated_length": 697.7862548828125, + "completions/min_length": 173.0, + "completions/min_terminated_length": 173.0, + "epoch": 0.4376546897670052, + "grad_norm": 2.648240804672241, + "kl": 5.7734375, + "learning_rate": 7.221456302584202e-07, + "loss": 0.3839, + "num_tokens": 734254277.0, + "reward": 1.78564453125, + "reward_std": 0.5704188346862793, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.822265625, + "rewards/format_reward/std": 0.3826628625392914, + "rewards/tag_count_reward/mean": 0.91064453125, + "rewards/tag_count_reward/std": 0.21203739941120148, + "step": 1282 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.056640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1829.0, + "completions/mean_length": 827.361328125, + "completions/mean_terminated_length": 754.0724487304688, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "epoch": 0.437996074080396, + "grad_norm": 1.8201979398727417, + "kl": 5.515625, + "learning_rate": 7.216501784916032e-07, + "loss": 0.3247, + "num_tokens": 734752654.0, + "reward": 1.71630859375, + "reward_std": 0.5852100849151611, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.78515625, + "rewards/format_reward/std": 0.4111155867576599, + "rewards/tag_count_reward/mean": 0.88818359375, + "rewards/tag_count_reward/std": 0.23160721361637115, + "step": 1283 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 767.416015625, + "completions/mean_terminated_length": 709.9203491210938, + "completions/min_length": 135.0, + "completions/min_terminated_length": 135.0, + "epoch": 0.4383374583937868, + "grad_norm": 2.1691770553588867, + "kl": 5.828125, + "learning_rate": 7.211544830987533e-07, + "loss": 0.4291, + "num_tokens": 735235955.0, + "reward": 1.826171875, + "reward_std": 0.5387592315673828, + "rewards/accuracy_reward/mean": 0.06653226166963577, + "rewards/accuracy_reward/std": 0.2494617998600006, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.916015625, + "rewards/tag_count_reward/std": 0.2081446796655655, + "step": 1284 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1964.0, + "completions/mean_length": 839.587890625, + "completions/mean_terminated_length": 782.75048828125, + "completions/min_length": 167.0, + "completions/min_terminated_length": 167.0, + "epoch": 0.4386788427071776, + "grad_norm": 1.0244237184524536, + "kl": 7.5546875, + "learning_rate": 7.206585447834188e-07, + "loss": 0.4849, + "num_tokens": 735737504.0, + "reward": 1.7470703125, + "reward_std": 0.6523882150650024, + "rewards/accuracy_reward/mean": 0.06653226166963577, + "rewards/accuracy_reward/std": 0.2494617998600006, + "rewards/format_reward/mean": 0.794921875, + "rewards/format_reward/std": 0.4041535556316376, + "rewards/tag_count_reward/mean": 0.8876953125, + "rewards/tag_count_reward/std": 0.23530170321464539, + "step": 1285 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 788.748046875, + "completions/mean_terminated_length": 748.1270141601562, + "completions/min_length": 122.0, + "completions/min_terminated_length": 122.0, + "epoch": 0.4390202270205684, + "grad_norm": 1.1738215684890747, + "kl": 6.421875, + "learning_rate": 7.201623642494943e-07, + "loss": 0.3847, + "num_tokens": 736220559.0, + "reward": 1.77685546875, + "reward_std": 0.5522069931030273, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.8125, + "rewards/format_reward/std": 0.39069411158561707, + "rewards/tag_count_reward/mean": 0.91162109375, + "rewards/tag_count_reward/std": 0.2036283165216446, + "step": 1286 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1848.0, + "completions/mean_length": 762.56640625, + "completions/mean_terminated_length": 721.1007690429688, + "completions/min_length": 34.0, + "completions/min_terminated_length": 34.0, + "epoch": 0.4393616113339592, + "grad_norm": 2.3395161628723145, + "kl": 6.890625, + "learning_rate": 7.196659422012166e-07, + "loss": 0.3967, + "num_tokens": 736686993.0, + "reward": 1.77490234375, + "reward_std": 0.5984107851982117, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.802734375, + "rewards/format_reward/std": 0.3983237147331238, + "rewards/tag_count_reward/mean": 0.91357421875, + "rewards/tag_count_reward/std": 0.20386749505996704, + "step": 1287 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1885.0, + "completions/mean_length": 684.34765625, + "completions/mean_terminated_length": 659.9483032226562, + "completions/min_length": 188.0, + "completions/min_terminated_length": 188.0, + "epoch": 0.43970299564735, + "grad_norm": 2.5489094257354736, + "kl": 5.8671875, + "learning_rate": 7.191692793431663e-07, + "loss": 0.3417, + "num_tokens": 737116659.0, + "reward": 1.865234375, + "reward_std": 0.4733661413192749, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.859375, + "rewards/format_reward/std": 0.3479743003845215, + "rewards/tag_count_reward/mean": 0.9296875, + "rewards/tag_count_reward/std": 0.180202916264534, + "step": 1288 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 777.376953125, + "completions/mean_terminated_length": 723.0325927734375, + "completions/min_length": 150.0, + "completions/min_terminated_length": 150.0, + "epoch": 0.4400443799607408, + "grad_norm": 1.7061121463775635, + "kl": 7.9921875, + "learning_rate": 7.186723763802654e-07, + "loss": 0.5473, + "num_tokens": 737590532.0, + "reward": 1.763671875, + "reward_std": 0.5881055593490601, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.8125, + "rewards/format_reward/std": 0.39069411158561707, + "rewards/tag_count_reward/mean": 0.908203125, + "rewards/tag_count_reward/std": 0.21127952635288239, + "step": 1289 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1968.0, + "completions/mean_length": 720.234375, + "completions/mean_terminated_length": 685.643310546875, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "epoch": 0.4403857642741316, + "grad_norm": 1.4125784635543823, + "kl": 6.5546875, + "learning_rate": 7.181752340177769e-07, + "loss": 0.396, + "num_tokens": 738035068.0, + "reward": 1.77880859375, + "reward_std": 0.5477635860443115, + "rewards/accuracy_reward/mean": 0.03125, + "rewards/accuracy_reward/std": 0.17416280508041382, + "rewards/format_reward/mean": 0.828125, + "rewards/format_reward/std": 0.3776407241821289, + "rewards/tag_count_reward/mean": 0.91943359375, + "rewards/tag_count_reward/std": 0.1869737207889557, + "step": 1290 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2029.0, + "completions/mean_length": 764.392578125, + "completions/mean_terminated_length": 725.65185546875, + "completions/min_length": 18.0, + "completions/min_terminated_length": 18.0, + "epoch": 0.4407271485875224, + "grad_norm": 1.820510745048523, + "kl": 5.8515625, + "learning_rate": 7.176778529613035e-07, + "loss": 0.373, + "num_tokens": 738512837.0, + "reward": 1.7880859375, + "reward_std": 0.5815557241439819, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.798828125, + "rewards/format_reward/std": 0.4012683033943176, + "rewards/tag_count_reward/mean": 0.9169921875, + "rewards/tag_count_reward/std": 0.19077126681804657, + "step": 1291 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1970.0, + "completions/mean_length": 819.72265625, + "completions/mean_terminated_length": 761.950927734375, + "completions/min_length": 192.0, + "completions/min_terminated_length": 192.0, + "epoch": 0.4410685329009132, + "grad_norm": 3.221315383911133, + "kl": 7.8125, + "learning_rate": 7.171802339167864e-07, + "loss": 0.4631, + "num_tokens": 739009815.0, + "reward": 1.81591796875, + "reward_std": 0.6045268177986145, + "rewards/accuracy_reward/mean": 0.103515625, + "rewards/accuracy_reward/std": 0.30492907762527466, + "rewards/format_reward/mean": 0.80859375, + "rewards/format_reward/std": 0.3937928080558777, + "rewards/tag_count_reward/mean": 0.90380859375, + "rewards/tag_count_reward/std": 0.21649256348609924, + "step": 1292 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 836.876953125, + "completions/mean_terminated_length": 774.704345703125, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "epoch": 0.441409917214304, + "grad_norm": 1.7177226543426514, + "kl": 7.7734375, + "learning_rate": 7.16682377590505e-07, + "loss": 0.4845, + "num_tokens": 739515048.0, + "reward": 1.744140625, + "reward_std": 0.6319704055786133, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.779296875, + "rewards/format_reward/std": 0.4151262938976288, + "rewards/tag_count_reward/mean": 0.8984375, + "rewards/tag_count_reward/std": 0.21993927657604218, + "step": 1293 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1799.0, + "completions/mean_length": 725.666015625, + "completions/mean_terminated_length": 699.32470703125, + "completions/min_length": 144.0, + "completions/min_terminated_length": 144.0, + "epoch": 0.4417513015276948, + "grad_norm": 1.3130954504013062, + "kl": 5.3203125, + "learning_rate": 7.161842846890751e-07, + "loss": 0.314, + "num_tokens": 739961325.0, + "reward": 1.79931640625, + "reward_std": 0.5158872604370117, + "rewards/accuracy_reward/mean": 0.037109375, + "rewards/accuracy_reward/std": 0.18921469151973724, + "rewards/format_reward/mean": 0.8359375, + "rewards/format_reward/std": 0.37069445848464966, + "rewards/tag_count_reward/mean": 0.92626953125, + "rewards/tag_count_reward/std": 0.18322066962718964, + "step": 1294 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1979.0, + "completions/mean_length": 787.35546875, + "completions/mean_terminated_length": 733.4379272460938, + "completions/min_length": 4.0, + "completions/min_terminated_length": 4.0, + "epoch": 0.4420926858410856, + "grad_norm": 1.687428593635559, + "kl": 5.6015625, + "learning_rate": 7.156859559194488e-07, + "loss": 0.357, + "num_tokens": 740440867.0, + "reward": 1.78564453125, + "reward_std": 0.5600343942642212, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.80859375, + "rewards/format_reward/std": 0.3937928080558777, + "rewards/tag_count_reward/mean": 0.92041015625, + "rewards/tag_count_reward/std": 0.19380934536457062, + "step": 1295 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 773.626953125, + "completions/mean_terminated_length": 740.4268798828125, + "completions/min_length": 159.0, + "completions/min_terminated_length": 159.0, + "epoch": 0.4424340701544764, + "grad_norm": 1.3360075950622559, + "kl": 6.4375, + "learning_rate": 7.151873919889122e-07, + "loss": 0.4097, + "num_tokens": 740915060.0, + "reward": 1.7626953125, + "reward_std": 0.5476801991462708, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.798828125, + "rewards/format_reward/std": 0.4012683033943176, + "rewards/tag_count_reward/mean": 0.9111328125, + "rewards/tag_count_reward/std": 0.20787595212459564, + "step": 1296 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1965.0, + "completions/mean_length": 745.232421875, + "completions/mean_terminated_length": 708.6083984375, + "completions/min_length": 31.0, + "completions/min_terminated_length": 31.0, + "epoch": 0.4427754544678672, + "grad_norm": 1.0148136615753174, + "kl": 6.0703125, + "learning_rate": 7.146885936050861e-07, + "loss": 0.3841, + "num_tokens": 741372347.0, + "reward": 1.78076171875, + "reward_std": 0.5385444164276123, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.818359375, + "rewards/format_reward/std": 0.38592514395713806, + "rewards/tag_count_reward/mean": 0.91943359375, + "rewards/tag_count_reward/std": 0.19901487231254578, + "step": 1297 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2002.0, + "completions/mean_length": 817.35546875, + "completions/mean_terminated_length": 780.2132568359375, + "completions/min_length": 82.0, + "completions/min_terminated_length": 82.0, + "epoch": 0.443116838781258, + "grad_norm": 2.377429723739624, + "kl": 5.15625, + "learning_rate": 7.141895614759232e-07, + "loss": 0.3373, + "num_tokens": 741868353.0, + "reward": 1.845703125, + "reward_std": 0.5714106559753418, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.841796875, + "rewards/format_reward/std": 0.36528825759887695, + "rewards/tag_count_reward/mean": 0.923828125, + "rewards/tag_count_reward/std": 0.19234009087085724, + "step": 1298 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1875.0, + "completions/mean_length": 756.16796875, + "completions/mean_terminated_length": 703.6544189453125, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "epoch": 0.4434582230946488, + "grad_norm": 2.2262306213378906, + "kl": 7.6796875, + "learning_rate": 7.136902963097085e-07, + "loss": 0.4547, + "num_tokens": 742327895.0, + "reward": 1.767578125, + "reward_std": 0.5512421131134033, + "rewards/accuracy_reward/mean": 0.038306452333927155, + "rewards/accuracy_reward/std": 0.19212885200977325, + "rewards/format_reward/mean": 0.82421875, + "rewards/format_reward/std": 0.3810062110424042, + "rewards/tag_count_reward/mean": 0.90625, + "rewards/tag_count_reward/std": 0.2178439050912857, + "step": 1299 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1947.0, + "completions/mean_length": 726.76953125, + "completions/mean_terminated_length": 689.62646484375, + "completions/min_length": 121.0, + "completions/min_terminated_length": 121.0, + "epoch": 0.4437996074080396, + "grad_norm": 3.62125301361084, + "kl": 9.34375, + "learning_rate": 7.131907988150575e-07, + "loss": 0.5471, + "num_tokens": 742778865.0, + "reward": 1.75634765625, + "reward_std": 0.671599805355072, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.775390625, + "rewards/format_reward/std": 0.41773295402526855, + "rewards/tag_count_reward/mean": 0.88916015625, + "rewards/tag_count_reward/std": 0.2315494418144226, + "step": 1300 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 869.12890625, + "completions/mean_terminated_length": 790.5375366210938, + "completions/min_length": 203.0, + "completions/min_terminated_length": 203.0, + "epoch": 0.4441409917214304, + "grad_norm": 3.7968902587890625, + "kl": 9.2890625, + "learning_rate": 7.126910697009156e-07, + "loss": 0.5645, + "num_tokens": 743310179.0, + "reward": 1.72314453125, + "reward_std": 0.6497185230255127, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.751953125, + "rewards/format_reward/std": 0.4323015511035919, + "rewards/tag_count_reward/mean": 0.88525390625, + "rewards/tag_count_reward/std": 0.23542095720767975, + "step": 1301 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 753.142578125, + "completions/mean_terminated_length": 724.7125854492188, + "completions/min_length": 78.0, + "completions/min_terminated_length": 78.0, + "epoch": 0.4444823760348212, + "grad_norm": 1.2830666303634644, + "kl": 6.3125, + "learning_rate": 7.121911096765571e-07, + "loss": 0.3837, + "num_tokens": 743777484.0, + "reward": 1.8046875, + "reward_std": 0.5349550247192383, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.826171875, + "rewards/format_reward/std": 0.3793322443962097, + "rewards/tag_count_reward/mean": 0.923828125, + "rewards/tag_count_reward/std": 0.19234009087085724, + "step": 1302 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1978.0, + "completions/mean_length": 770.044921875, + "completions/mean_terminated_length": 723.4797973632812, + "completions/min_length": 219.0, + "completions/min_terminated_length": 219.0, + "epoch": 0.444823760348212, + "grad_norm": 1.2163110971450806, + "kl": 6.5078125, + "learning_rate": 7.116909194515831e-07, + "loss": 0.4312, + "num_tokens": 744247491.0, + "reward": 1.83203125, + "reward_std": 0.5182892680168152, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.857421875, + "rewards/format_reward/std": 0.3499840497970581, + "rewards/tag_count_reward/mean": 0.9296875, + "rewards/tag_count_reward/std": 0.18946701288223267, + "step": 1303 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2010.0, + "completions/mean_length": 757.763671875, + "completions/mean_terminated_length": 710.7510375976562, + "completions/min_length": 218.0, + "completions/min_terminated_length": 218.0, + "epoch": 0.4451651446616028, + "grad_norm": 1.347507357597351, + "kl": 7.1015625, + "learning_rate": 7.111904997359229e-07, + "loss": 0.4555, + "num_tokens": 744711338.0, + "reward": 1.80615234375, + "reward_std": 0.543106198310852, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.830078125, + "rewards/format_reward/std": 0.3759314715862274, + "rewards/tag_count_reward/mean": 0.92333984375, + "rewards/tag_count_reward/std": 0.20237624645233154, + "step": 1304 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2030.0, + "completions/mean_length": 790.521484375, + "completions/mean_terminated_length": 736.7393188476562, + "completions/min_length": 70.0, + "completions/min_terminated_length": 70.0, + "epoch": 0.4455065289749936, + "grad_norm": 1.7766551971435547, + "kl": 6.9296875, + "learning_rate": 7.106898512398305e-07, + "loss": 0.461, + "num_tokens": 745189237.0, + "reward": 1.8125, + "reward_std": 0.572502851486206, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.822265625, + "rewards/format_reward/std": 0.3826628625392914, + "rewards/tag_count_reward/mean": 0.916015625, + "rewards/tag_count_reward/std": 0.2069661170244217, + "step": 1305 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2006.0, + "completions/mean_length": 724.638671875, + "completions/mean_terminated_length": 665.222412109375, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "epoch": 0.4458479132883844, + "grad_norm": 2.5597076416015625, + "kl": 6.390625, + "learning_rate": 7.101889746738848e-07, + "loss": 0.4223, + "num_tokens": 745635292.0, + "reward": 1.82275390625, + "reward_std": 0.5397883653640747, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.92626953125, + "rewards/tag_count_reward/std": 0.19859671592712402, + "step": 1306 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1945.0, + "completions/mean_length": 803.638671875, + "completions/mean_terminated_length": 753.0548706054688, + "completions/min_length": 13.0, + "completions/min_terminated_length": 13.0, + "epoch": 0.4461892976017752, + "grad_norm": 1.1304194927215576, + "kl": 8.203125, + "learning_rate": 7.096878707489885e-07, + "loss": 0.518, + "num_tokens": 746127459.0, + "reward": 1.7470703125, + "reward_std": 0.6205352544784546, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.79296875, + "rewards/format_reward/std": 0.40557438135147095, + "rewards/tag_count_reward/mean": 0.8994140625, + "rewards/tag_count_reward/std": 0.22369353473186493, + "step": 1307 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2011.0, + "completions/mean_length": 788.05078125, + "completions/mean_terminated_length": 747.4072265625, + "completions/min_length": 172.0, + "completions/min_terminated_length": 172.0, + "epoch": 0.446530681915166, + "grad_norm": 1.2309277057647705, + "kl": 6.25, + "learning_rate": 7.091865401763671e-07, + "loss": 0.381, + "num_tokens": 746606573.0, + "reward": 1.84423828125, + "reward_std": 0.5464041233062744, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.92626953125, + "rewards/tag_count_reward/std": 0.18848544359207153, + "step": 1308 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2015.0, + "completions/mean_length": 772.6796875, + "completions/mean_terminated_length": 715.4203491210938, + "completions/min_length": 13.0, + "completions/min_terminated_length": 13.0, + "epoch": 0.4468720662285568, + "grad_norm": 2.441206693649292, + "kl": 8.65625, + "learning_rate": 7.086849836675675e-07, + "loss": 0.5249, + "num_tokens": 747076921.0, + "reward": 1.76611328125, + "reward_std": 0.6479424238204956, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.78125, + "rewards/format_reward/std": 0.41380295157432556, + "rewards/tag_count_reward/mean": 0.89306640625, + "rewards/tag_count_reward/std": 0.22429661452770233, + "step": 1309 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1767.0, + "completions/mean_length": 767.103515625, + "completions/mean_terminated_length": 706.8568115234375, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.4472134505419476, + "grad_norm": 2.090247392654419, + "kl": 7.59375, + "learning_rate": 7.081832019344573e-07, + "loss": 0.4598, + "num_tokens": 747558638.0, + "reward": 1.74658203125, + "reward_std": 0.5532182455062866, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.80078125, + "rewards/format_reward/std": 0.39980348944664, + "rewards/tag_count_reward/mean": 0.90283203125, + "rewards/tag_count_reward/std": 0.21491996943950653, + "step": 1310 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.064453125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1998.0, + "completions/mean_length": 916.376953125, + "completions/mean_terminated_length": 838.4154663085938, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "epoch": 0.4475548348553384, + "grad_norm": 2.8132505416870117, + "kl": 8.328125, + "learning_rate": 7.076811956892241e-07, + "loss": 0.4943, + "num_tokens": 748113839.0, + "reward": 1.6796875, + "reward_std": 0.6440863609313965, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.7421875, + "rewards/format_reward/std": 0.43785804510116577, + "rewards/tag_count_reward/mean": 0.87890625, + "rewards/tag_count_reward/std": 0.235612154006958, + "step": 1311 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2008.0, + "completions/mean_length": 786.80078125, + "completions/mean_terminated_length": 746.116943359375, + "completions/min_length": 126.0, + "completions/min_terminated_length": 126.0, + "epoch": 0.4478962191687292, + "grad_norm": 1.1374046802520752, + "kl": 5.7890625, + "learning_rate": 7.07178965644374e-07, + "loss": 0.3638, + "num_tokens": 748599785.0, + "reward": 1.85693359375, + "reward_std": 0.5871137380599976, + "rewards/accuracy_reward/mean": 0.107421875, + "rewards/accuracy_reward/std": 0.30995169281959534, + "rewards/format_reward/mean": 0.82421875, + "rewards/format_reward/std": 0.3810062110424042, + "rewards/tag_count_reward/mean": 0.92529296875, + "rewards/tag_count_reward/std": 0.1861388236284256, + "step": 1312 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2017.0, + "completions/mean_length": 726.25, + "completions/mean_terminated_length": 683.6128540039062, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "epoch": 0.44823760348212, + "grad_norm": 2.2175955772399902, + "kl": 5.7265625, + "learning_rate": 7.066765125127305e-07, + "loss": 0.3893, + "num_tokens": 749043593.0, + "reward": 1.81494140625, + "reward_std": 0.5199028253555298, + "rewards/accuracy_reward/mean": 0.03125, + "rewards/accuracy_reward/std": 0.17416280508041382, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.93212890625, + "rewards/tag_count_reward/std": 0.1874433010816574, + "step": 1313 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1924.0, + "completions/mean_length": 733.255859375, + "completions/mean_terminated_length": 707.0657348632812, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "epoch": 0.4485789877955108, + "grad_norm": 3.9274680614471436, + "kl": 4.7578125, + "learning_rate": 7.061738370074342e-07, + "loss": 0.3534, + "num_tokens": 749498892.0, + "reward": 1.88720703125, + "reward_std": 0.5058292150497437, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.94189453125, + "rewards/tag_count_reward/std": 0.16893739998340607, + "step": 1314 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1890.0, + "completions/mean_length": 842.671875, + "completions/mean_terminated_length": 767.6514892578125, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "epoch": 0.4489203721089016, + "grad_norm": 1.1414005756378174, + "kl": 8.15625, + "learning_rate": 7.056709398419407e-07, + "loss": 0.5243, + "num_tokens": 750019828.0, + "reward": 1.69921875, + "reward_std": 0.6134747266769409, + "rewards/accuracy_reward/mean": 0.0234375, + "rewards/accuracy_reward/std": 0.15143637359142303, + "rewards/format_reward/mean": 0.78515625, + "rewards/format_reward/std": 0.4111155867576599, + "rewards/tag_count_reward/mean": 0.890625, + "rewards/tag_count_reward/std": 0.23355920612812042, + "step": 1315 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2014.0, + "completions/mean_length": 797.68359375, + "completions/mean_terminated_length": 744.207763671875, + "completions/min_length": 11.0, + "completions/min_terminated_length": 11.0, + "epoch": 0.4492617564222924, + "grad_norm": 1.8736921548843384, + "kl": 6.4765625, + "learning_rate": 7.051678217300207e-07, + "loss": 0.3547, + "num_tokens": 750511106.0, + "reward": 1.78125, + "reward_std": 0.60211181640625, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.80078125, + "rewards/format_reward/std": 0.39980348944664, + "rewards/tag_count_reward/mean": 0.90234375, + "rewards/tag_count_reward/std": 0.21892902255058289, + "step": 1316 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1933.0, + "completions/mean_length": 793.58203125, + "completions/mean_terminated_length": 734.5807495117188, + "completions/min_length": 162.0, + "completions/min_terminated_length": 162.0, + "epoch": 0.4496031407356832, + "grad_norm": 1.5235521793365479, + "kl": 7.015625, + "learning_rate": 7.046644833857583e-07, + "loss": 0.4618, + "num_tokens": 750989820.0, + "reward": 1.791015625, + "reward_std": 0.5913950800895691, + "rewards/accuracy_reward/mean": 0.07258064299821854, + "rewards/accuracy_reward/std": 0.25970885157585144, + "rewards/format_reward/mean": 0.81640625, + "rewards/format_reward/std": 0.3875311613082886, + "rewards/tag_count_reward/mean": 0.904296875, + "rewards/tag_count_reward/std": 0.21642689406871796, + "step": 1317 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 823.259765625, + "completions/mean_terminated_length": 770.8778686523438, + "completions/min_length": 218.0, + "completions/min_terminated_length": 218.0, + "epoch": 0.449944525049074, + "grad_norm": 1.2474850416183472, + "kl": 6.9609375, + "learning_rate": 7.041609255235503e-07, + "loss": 0.4429, + "num_tokens": 751484241.0, + "reward": 1.783203125, + "reward_std": 0.517959713935852, + "rewards/accuracy_reward/mean": 0.029296875, + "rewards/accuracy_reward/std": 0.16880230605602264, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.916015625, + "rewards/tag_count_reward/std": 0.2045886218547821, + "step": 1318 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2004.0, + "completions/mean_length": 778.888671875, + "completions/mean_terminated_length": 735.3030395507812, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "epoch": 0.4502859093624648, + "grad_norm": 1.5461757183074951, + "kl": 6.10546875, + "learning_rate": 7.036571488581049e-07, + "loss": 0.3639, + "num_tokens": 751962616.0, + "reward": 1.76123046875, + "reward_std": 0.5829464197158813, + "rewards/accuracy_reward/mean": 0.04233871027827263, + "rewards/accuracy_reward/std": 0.2015640139579773, + "rewards/format_reward/mean": 0.810546875, + "rewards/format_reward/std": 0.3922513723373413, + "rewards/tag_count_reward/mean": 0.90966796875, + "rewards/tag_count_reward/std": 0.21334922313690186, + "step": 1319 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 818.220703125, + "completions/mean_terminated_length": 757.7396850585938, + "completions/min_length": 151.0, + "completions/min_terminated_length": 151.0, + "epoch": 0.4506272936758556, + "grad_norm": 1.982473611831665, + "kl": 7.2734375, + "learning_rate": 7.031531541044411e-07, + "loss": 0.4438, + "num_tokens": 752449673.0, + "reward": 1.80126953125, + "reward_std": 0.5766713619232178, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.828125, + "rewards/format_reward/std": 0.3776407241821289, + "rewards/tag_count_reward/mean": 0.90673828125, + "rewards/tag_count_reward/std": 0.22056347131729126, + "step": 1320 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.056640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2013.0, + "completions/mean_length": 798.314453125, + "completions/mean_terminated_length": 723.2816162109375, + "completions/min_length": 138.0, + "completions/min_terminated_length": 138.0, + "epoch": 0.4509686779892464, + "grad_norm": 1.7228455543518066, + "kl": 7.1796875, + "learning_rate": 7.026489419778871e-07, + "loss": 0.5007, + "num_tokens": 752939882.0, + "reward": 1.79052734375, + "reward_std": 0.611250102519989, + "rewards/accuracy_reward/mean": 0.08669354766607285, + "rewards/accuracy_reward/std": 0.281669557094574, + "rewards/format_reward/mean": 0.8046875, + "rewards/format_reward/std": 0.3968288004398346, + "rewards/tag_count_reward/mean": 0.90185546875, + "rewards/tag_count_reward/std": 0.22504940629005432, + "step": 1321 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.064453125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2011.0, + "completions/mean_length": 867.53515625, + "completions/mean_terminated_length": 786.2088012695312, + "completions/min_length": 173.0, + "completions/min_terminated_length": 173.0, + "epoch": 0.4513100623026372, + "grad_norm": 2.166879653930664, + "kl": 6.7265625, + "learning_rate": 7.021445131940797e-07, + "loss": 0.4772, + "num_tokens": 753456572.0, + "reward": 1.77001953125, + "reward_std": 0.587213933467865, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.810546875, + "rewards/format_reward/std": 0.3922513723373413, + "rewards/tag_count_reward/mean": 0.90673828125, + "rewards/tag_count_reward/std": 0.21208246052265167, + "step": 1322 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2028.0, + "completions/mean_length": 824.8515625, + "completions/mean_terminated_length": 772.5377197265625, + "completions/min_length": 189.0, + "completions/min_terminated_length": 189.0, + "epoch": 0.451651446616028, + "grad_norm": 2.0856263637542725, + "kl": 7.90625, + "learning_rate": 7.016398684689636e-07, + "loss": 0.5179, + "num_tokens": 753964992.0, + "reward": 1.76171875, + "reward_std": 0.613147497177124, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.78515625, + "rewards/format_reward/std": 0.4111155867576599, + "rewards/tag_count_reward/mean": 0.896484375, + "rewards/tag_count_reward/std": 0.21564844250679016, + "step": 1323 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 780.615234375, + "completions/mean_terminated_length": 734.4352416992188, + "completions/min_length": 56.0, + "completions/min_terminated_length": 56.0, + "epoch": 0.4519928309294188, + "grad_norm": 1.2203400135040283, + "kl": 6.7265625, + "learning_rate": 7.011350085187895e-07, + "loss": 0.4204, + "num_tokens": 754435643.0, + "reward": 1.77294921875, + "reward_std": 0.6677350997924805, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.78515625, + "rewards/format_reward/std": 0.4111155867576599, + "rewards/tag_count_reward/mean": 0.89013671875, + "rewards/tag_count_reward/std": 0.23515698313713074, + "step": 1324 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 804.119140625, + "completions/mean_terminated_length": 745.6134643554688, + "completions/min_length": 187.0, + "completions/min_terminated_length": 187.0, + "epoch": 0.4523342152428096, + "grad_norm": 2.034092903137207, + "kl": 6.734375, + "learning_rate": 7.006299340601136e-07, + "loss": 0.4632, + "num_tokens": 754927608.0, + "reward": 1.78076171875, + "reward_std": 0.5564095377922058, + "rewards/accuracy_reward/mean": 0.04233871027827263, + "rewards/accuracy_reward/std": 0.2015640139579773, + "rewards/format_reward/mean": 0.826171875, + "rewards/format_reward/std": 0.3793322443962097, + "rewards/tag_count_reward/mean": 0.91357421875, + "rewards/tag_count_reward/std": 0.20743593573570251, + "step": 1325 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1951.0, + "completions/mean_length": 747.501953125, + "completions/mean_terminated_length": 694.6361694335938, + "completions/min_length": 15.0, + "completions/min_terminated_length": 15.0, + "epoch": 0.4526755995562004, + "grad_norm": 1.7306252717971802, + "kl": 7.0546875, + "learning_rate": 7.001246458097972e-07, + "loss": 0.4403, + "num_tokens": 755387049.0, + "reward": 1.87548828125, + "reward_std": 0.5507863163948059, + "rewards/accuracy_reward/mean": 0.11895161122083664, + "rewards/accuracy_reward/std": 0.3240584135055542, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.91650390625, + "rewards/tag_count_reward/std": 0.20804768800735474, + "step": 1326 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1858.0, + "completions/mean_length": 798.201171875, + "completions/mean_terminated_length": 750.0344848632812, + "completions/min_length": 177.0, + "completions/min_terminated_length": 177.0, + "epoch": 0.4530169838695912, + "grad_norm": 1.4104769229888916, + "kl": 6.40625, + "learning_rate": 6.99619144485004e-07, + "loss": 0.4229, + "num_tokens": 755873392.0, + "reward": 1.8349609375, + "reward_std": 0.5700967907905579, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.8359375, + "rewards/format_reward/std": 0.37069445848464966, + "rewards/tag_count_reward/mean": 0.9208984375, + "rewards/tag_count_reward/std": 0.19929614663124084, + "step": 1327 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1963.0, + "completions/mean_length": 790.73828125, + "completions/mean_terminated_length": 747.5596313476562, + "completions/min_length": 127.0, + "completions/min_terminated_length": 127.0, + "epoch": 0.453358368182982, + "grad_norm": 3.7157509326934814, + "kl": 4.65625, + "learning_rate": 6.99113430803201e-07, + "loss": 0.3258, + "num_tokens": 756362346.0, + "reward": 1.84326171875, + "reward_std": 0.5458806753158569, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.92333984375, + "rewards/tag_count_reward/std": 0.19246363639831543, + "step": 1328 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1955.0, + "completions/mean_length": 839.15625, + "completions/mean_terminated_length": 807.663330078125, + "completions/min_length": 157.0, + "completions/min_terminated_length": 157.0, + "epoch": 0.4536997524963728, + "grad_norm": 1.4480165243148804, + "kl": 6.3984375, + "learning_rate": 6.986075054821561e-07, + "loss": 0.3843, + "num_tokens": 756869130.0, + "reward": 1.783203125, + "reward_std": 0.5669708251953125, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.814453125, + "rewards/format_reward/std": 0.38912075757980347, + "rewards/tag_count_reward/mean": 0.919921875, + "rewards/tag_count_reward/std": 0.19138385355472565, + "step": 1329 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.056640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2011.0, + "completions/mean_length": 827.486328125, + "completions/mean_terminated_length": 754.2049560546875, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "epoch": 0.4540411368097636, + "grad_norm": 2.614987373352051, + "kl": 9.59375, + "learning_rate": 6.98101369239938e-07, + "loss": 0.6153, + "num_tokens": 757356915.0, + "reward": 1.80810546875, + "reward_std": 0.5938037633895874, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.826171875, + "rewards/format_reward/std": 0.3793322443962097, + "rewards/tag_count_reward/mean": 0.90576171875, + "rewards/tag_count_reward/std": 0.21903319656848907, + "step": 1330 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2005.0, + "completions/mean_length": 817.677734375, + "completions/mean_terminated_length": 751.8580322265625, + "completions/min_length": 156.0, + "completions/min_terminated_length": 156.0, + "epoch": 0.4543825211231544, + "grad_norm": 1.9934707880020142, + "kl": 8.5703125, + "learning_rate": 6.975950227949143e-07, + "loss": 0.5526, + "num_tokens": 757857726.0, + "reward": 1.82568359375, + "reward_std": 0.619998037815094, + "rewards/accuracy_reward/mean": 0.111328125, + "rewards/accuracy_reward/std": 0.31484565138816833, + "rewards/format_reward/mean": 0.8125, + "rewards/format_reward/std": 0.39069411158561707, + "rewards/tag_count_reward/mean": 0.90185546875, + "rewards/tag_count_reward/std": 0.22286489605903625, + "step": 1331 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1869.0, + "completions/mean_length": 841.33203125, + "completions/mean_terminated_length": 789.7230834960938, + "completions/min_length": 194.0, + "completions/min_terminated_length": 194.0, + "epoch": 0.4547239054365452, + "grad_norm": 1.104121446609497, + "kl": 8.2109375, + "learning_rate": 6.970884668657512e-07, + "loss": 0.5149, + "num_tokens": 758371464.0, + "reward": 1.76904296875, + "reward_std": 0.540198564529419, + "rewards/accuracy_reward/mean": 0.029296875, + "rewards/accuracy_reward/std": 0.16880230605602264, + "rewards/format_reward/mean": 0.8359375, + "rewards/format_reward/std": 0.37069445848464966, + "rewards/tag_count_reward/mean": 0.90380859375, + "rewards/tag_count_reward/std": 0.22316910326480865, + "step": 1332 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2014.0, + "completions/mean_length": 844.0859375, + "completions/mean_terminated_length": 782.2833862304688, + "completions/min_length": 173.0, + "completions/min_terminated_length": 173.0, + "epoch": 0.455065289749936, + "grad_norm": 3.675091028213501, + "kl": 8.7109375, + "learning_rate": 6.965817021714124e-07, + "loss": 0.5246, + "num_tokens": 758886868.0, + "reward": 1.77783203125, + "reward_std": 0.5871965885162354, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.806640625, + "rewards/format_reward/std": 0.39531853795051575, + "rewards/tag_count_reward/mean": 0.90478515625, + "rewards/tag_count_reward/std": 0.21972574293613434, + "step": 1333 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2020.0, + "completions/mean_length": 780.27734375, + "completions/mean_terminated_length": 747.25048828125, + "completions/min_length": 162.0, + "completions/min_terminated_length": 162.0, + "epoch": 0.4554066740633268, + "grad_norm": 1.0809215307235718, + "kl": 5.0703125, + "learning_rate": 6.960747294311575e-07, + "loss": 0.3025, + "num_tokens": 759361826.0, + "reward": 1.8271484375, + "reward_std": 0.5094544887542725, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.8359375, + "rewards/format_reward/std": 0.37069445848464966, + "rewards/tag_count_reward/mean": 0.9189453125, + "rewards/tag_count_reward/std": 0.1954032927751541, + "step": 1334 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 824.82421875, + "completions/mean_terminated_length": 759.3867797851562, + "completions/min_length": 162.0, + "completions/min_terminated_length": 162.0, + "epoch": 0.4557480583767176, + "grad_norm": 3.4591548442840576, + "kl": 6.4375, + "learning_rate": 6.955675493645415e-07, + "loss": 0.476, + "num_tokens": 759856440.0, + "reward": 1.8408203125, + "reward_std": 0.5953292846679688, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.82421875, + "rewards/format_reward/std": 0.3810062110424042, + "rewards/tag_count_reward/mean": 0.9189453125, + "rewards/tag_count_reward/std": 0.19973675906658173, + "step": 1335 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1973.0, + "completions/mean_length": 827.73046875, + "completions/mean_terminated_length": 788.3668823242188, + "completions/min_length": 96.0, + "completions/min_terminated_length": 96.0, + "epoch": 0.45608944269010837, + "grad_norm": 3.3444979190826416, + "kl": 5.34375, + "learning_rate": 6.950601626914139e-07, + "loss": 0.3554, + "num_tokens": 760362846.0, + "reward": 1.83544921875, + "reward_std": 0.5050147771835327, + "rewards/accuracy_reward/mean": 0.058467742055654526, + "rewards/accuracy_reward/std": 0.23486268520355225, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.92724609375, + "rewards/tag_count_reward/std": 0.19710436463356018, + "step": 1336 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1975.0, + "completions/mean_length": 823.904296875, + "completions/mean_terminated_length": 761.0657348632812, + "completions/min_length": 195.0, + "completions/min_terminated_length": 195.0, + "epoch": 0.4564308270034992, + "grad_norm": 4.18440580368042, + "kl": 5.53125, + "learning_rate": 6.94552570131917e-07, + "loss": 0.4259, + "num_tokens": 760864061.0, + "reward": 1.85498046875, + "reward_std": 0.5451841354370117, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.92138671875, + "rewards/tag_count_reward/std": 0.20040719211101532, + "step": 1337 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1980.0, + "completions/mean_length": 818.67578125, + "completions/mean_terminated_length": 781.5734252929688, + "completions/min_length": 148.0, + "completions/min_terminated_length": 148.0, + "epoch": 0.45677221131689, + "grad_norm": 2.1832258701324463, + "kl": 5.09375, + "learning_rate": 6.940447724064861e-07, + "loss": 0.3425, + "num_tokens": 761359943.0, + "reward": 1.9228515625, + "reward_std": 0.5376400947570801, + "rewards/accuracy_reward/mean": 0.119140625, + "rewards/accuracy_reward/std": 0.32427072525024414, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.9345703125, + "rewards/tag_count_reward/std": 0.18668025732040405, + "step": 1338 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2024.0, + "completions/mean_length": 800.65234375, + "completions/mean_terminated_length": 757.8141479492188, + "completions/min_length": 118.0, + "completions/min_terminated_length": 118.0, + "epoch": 0.4571135956302808, + "grad_norm": 1.333736538887024, + "kl": 6.3125, + "learning_rate": 6.935367702358469e-07, + "loss": 0.398, + "num_tokens": 761838021.0, + "reward": 1.90234375, + "reward_std": 0.5752283334732056, + "rewards/accuracy_reward/mean": 0.130859375, + "rewards/accuracy_reward/std": 0.33757632970809937, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.91796875, + "rewards/tag_count_reward/std": 0.2035866528749466, + "step": 1339 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1938.0, + "completions/mean_length": 755.560546875, + "completions/mean_terminated_length": 694.7709350585938, + "completions/min_length": 126.0, + "completions/min_terminated_length": 126.0, + "epoch": 0.45745497994367157, + "grad_norm": 1.5195820331573486, + "kl": 7.4921875, + "learning_rate": 6.930285643410154e-07, + "loss": 0.4774, + "num_tokens": 762306980.0, + "reward": 1.8505859375, + "reward_std": 0.5642558932304382, + "rewards/accuracy_reward/mean": 0.109375, + "rewards/accuracy_reward/std": 0.31241437792778015, + "rewards/format_reward/mean": 0.83203125, + "rewards/format_reward/std": 0.374204158782959, + "rewards/tag_count_reward/mean": 0.9091796875, + "rewards/tag_count_reward/std": 0.21908392012119293, + "step": 1340 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2004.0, + "completions/mean_length": 766.0390625, + "completions/mean_terminated_length": 719.3279418945312, + "completions/min_length": 200.0, + "completions/min_terminated_length": 200.0, + "epoch": 0.4577963642570624, + "grad_norm": 1.7313032150268555, + "kl": 6.7265625, + "learning_rate": 6.925201554432972e-07, + "loss": 0.4252, + "num_tokens": 762779560.0, + "reward": 1.830078125, + "reward_std": 0.4659072756767273, + "rewards/accuracy_reward/mean": 0.02916666679084301, + "rewards/accuracy_reward/std": 0.1684490591287613, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.931640625, + "rewards/tag_count_reward/std": 0.19400213658809662, + "step": 1341 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1913.0, + "completions/mean_length": 855.6640625, + "completions/mean_terminated_length": 804.6680908203125, + "completions/min_length": 188.0, + "completions/min_terminated_length": 188.0, + "epoch": 0.4581377485704532, + "grad_norm": 0.7581866979598999, + "kl": 6.7890625, + "learning_rate": 6.920115442642858e-07, + "loss": 0.4448, + "num_tokens": 763302764.0, + "reward": 1.828125, + "reward_std": 0.583230197429657, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.833984375, + "rewards/format_reward/std": 0.3724585771560669, + "rewards/tag_count_reward/mean": 0.919921875, + "rewards/tag_count_reward/std": 0.20074127614498138, + "step": 1342 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1976.0, + "completions/mean_length": 834.03125, + "completions/mean_terminated_length": 782.1100463867188, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "epoch": 0.458479132883844, + "grad_norm": 1.2648884057998657, + "kl": 8.03125, + "learning_rate": 6.915027315258614e-07, + "loss": 0.5089, + "num_tokens": 763802876.0, + "reward": 1.81884765625, + "reward_std": 0.5947195291519165, + "rewards/accuracy_reward/mean": 0.07661290466785431, + "rewards/accuracy_reward/std": 0.2662447690963745, + "rewards/format_reward/mean": 0.83203125, + "rewards/format_reward/std": 0.374204158782959, + "rewards/tag_count_reward/mean": 0.91259765625, + "rewards/tag_count_reward/std": 0.21285150945186615, + "step": 1343 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 783.857421875, + "completions/mean_terminated_length": 718.9630737304688, + "completions/min_length": 133.0, + "completions/min_terminated_length": 133.0, + "epoch": 0.45882051719723477, + "grad_norm": 1.795419692993164, + "kl": 7.671875, + "learning_rate": 6.909937179501908e-07, + "loss": 0.543, + "num_tokens": 764281123.0, + "reward": 1.830078125, + "reward_std": 0.5486209988594055, + "rewards/accuracy_reward/mean": 0.052419353276491165, + "rewards/accuracy_reward/std": 0.22309619188308716, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.923828125, + "rewards/tag_count_reward/std": 0.20286257565021515, + "step": 1344 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1993.0, + "completions/mean_length": 782.736328125, + "completions/mean_terminated_length": 741.9213256835938, + "completions/min_length": 5.0, + "completions/min_terminated_length": 5.0, + "epoch": 0.4591619015106256, + "grad_norm": 2.3222286701202393, + "kl": 7.6171875, + "learning_rate": 6.904845042597258e-07, + "loss": 0.4465, + "num_tokens": 764752572.0, + "reward": 1.82080078125, + "reward_std": 0.5536209344863892, + "rewards/accuracy_reward/mean": 0.08064515888690948, + "rewards/accuracy_reward/std": 0.2725643217563629, + "rewards/format_reward/mean": 0.828125, + "rewards/format_reward/std": 0.3776407241821289, + "rewards/tag_count_reward/mean": 0.91455078125, + "rewards/tag_count_reward/std": 0.20901454985141754, + "step": 1345 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2019.0, + "completions/mean_length": 854.87109375, + "completions/mean_terminated_length": 793.6221923828125, + "completions/min_length": 190.0, + "completions/min_terminated_length": 190.0, + "epoch": 0.4595032858240164, + "grad_norm": 1.380971908569336, + "kl": 6.8828125, + "learning_rate": 6.899750911772019e-07, + "loss": 0.4723, + "num_tokens": 765262410.0, + "reward": 1.8125, + "reward_std": 0.48659664392471313, + "rewards/accuracy_reward/mean": 0.025390625, + "rewards/accuracy_reward/std": 0.15746226906776428, + "rewards/format_reward/mean": 0.859375, + "rewards/format_reward/std": 0.3479743003845215, + "rewards/tag_count_reward/mean": 0.927734375, + "rewards/tag_count_reward/std": 0.20005464553833008, + "step": 1346 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2028.0, + "completions/mean_length": 815.16796875, + "completions/mean_terminated_length": 767.6551513671875, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "epoch": 0.4598446701374072, + "grad_norm": 1.618891954421997, + "kl": 6.0859375, + "learning_rate": 6.894654794256378e-07, + "loss": 0.4128, + "num_tokens": 765767392.0, + "reward": 1.84375, + "reward_std": 0.5409480333328247, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.92578125, + "rewards/tag_count_reward/std": 0.2005603015422821, + "step": 1347 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1762.0, + "completions/mean_length": 778.259765625, + "completions/mean_terminated_length": 737.3003540039062, + "completions/min_length": 163.0, + "completions/min_terminated_length": 163.0, + "epoch": 0.46018605445079797, + "grad_norm": 2.0402615070343018, + "kl": 5.5625, + "learning_rate": 6.889556697283344e-07, + "loss": 0.3458, + "num_tokens": 766239573.0, + "reward": 1.8759765625, + "reward_std": 0.45865148305892944, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.873046875, + "rewards/format_reward/std": 0.33324605226516724, + "rewards/tag_count_reward/mean": 0.9384765625, + "rewards/tag_count_reward/std": 0.18138417601585388, + "step": 1348 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1892.0, + "completions/mean_length": 784.96484375, + "completions/mean_terminated_length": 717.39501953125, + "completions/min_length": 198.0, + "completions/min_terminated_length": 198.0, + "epoch": 0.4605274387641888, + "grad_norm": 1.3208638429641724, + "kl": 7.9921875, + "learning_rate": 6.88445662808873e-07, + "loss": 0.5725, + "num_tokens": 766722515.0, + "reward": 1.84619140625, + "reward_std": 0.5656688213348389, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.859375, + "rewards/format_reward/std": 0.3479743003845215, + "rewards/tag_count_reward/mean": 0.92822265625, + "rewards/tag_count_reward/std": 0.19371071457862854, + "step": 1349 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.060546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2024.0, + "completions/mean_length": 853.98046875, + "completions/mean_terminated_length": 777.0270385742188, + "completions/min_length": 162.0, + "completions/min_terminated_length": 162.0, + "epoch": 0.4608688230775796, + "grad_norm": 2.3935322761535645, + "kl": 9.0, + "learning_rate": 6.879354593911154e-07, + "loss": 0.6046, + "num_tokens": 767239801.0, + "reward": 1.74853515625, + "reward_std": 0.5295427441596985, + "rewards/accuracy_reward/mean": 0.0078125, + "rewards/accuracy_reward/std": 0.08812850713729858, + "rewards/format_reward/mean": 0.82421875, + "rewards/format_reward/std": 0.3810062110424042, + "rewards/tag_count_reward/mean": 0.91650390625, + "rewards/tag_count_reward/std": 0.20804768800735474, + "step": 1350 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 818.095703125, + "completions/mean_terminated_length": 754.958984375, + "completions/min_length": 123.0, + "completions/min_terminated_length": 123.0, + "epoch": 0.4612102073909704, + "grad_norm": 3.591526985168457, + "kl": 8.703125, + "learning_rate": 6.874250601992019e-07, + "loss": 0.527, + "num_tokens": 767733882.0, + "reward": 1.80224609375, + "reward_std": 0.5414711833000183, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.822265625, + "rewards/format_reward/std": 0.3826628625392914, + "rewards/tag_count_reward/mean": 0.91552734375, + "rewards/tag_count_reward/std": 0.20228178799152374, + "step": 1351 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1972.0, + "completions/mean_length": 766.748046875, + "completions/mean_terminated_length": 700.9754028320312, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "epoch": 0.46155159170436116, + "grad_norm": 3.2490854263305664, + "kl": 7.140625, + "learning_rate": 6.869144659575507e-07, + "loss": 0.4527, + "num_tokens": 768203945.0, + "reward": 1.876953125, + "reward_std": 0.5580621957778931, + "rewards/accuracy_reward/mean": 0.130859375, + "rewards/accuracy_reward/std": 0.33757632970809937, + "rewards/format_reward/mean": 0.826171875, + "rewards/format_reward/std": 0.3793322443962097, + "rewards/tag_count_reward/mean": 0.919921875, + "rewards/tag_count_reward/std": 0.19265778362751007, + "step": 1352 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1835.0, + "completions/mean_length": 807.17578125, + "completions/mean_terminated_length": 761.9635620117188, + "completions/min_length": 209.0, + "completions/min_terminated_length": 209.0, + "epoch": 0.461892976017752, + "grad_norm": 1.5669411420822144, + "kl": 6.7890625, + "learning_rate": 6.864036773908572e-07, + "loss": 0.4174, + "num_tokens": 768703747.0, + "reward": 1.810546875, + "reward_std": 0.5165926218032837, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.833984375, + "rewards/format_reward/std": 0.3724585771560669, + "rewards/tag_count_reward/mean": 0.92578125, + "rewards/tag_count_reward/std": 0.19499453902244568, + "step": 1353 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1608.0, + "completions/mean_length": 783.302734375, + "completions/mean_terminated_length": 737.2206420898438, + "completions/min_length": 23.0, + "completions/min_terminated_length": 23.0, + "epoch": 0.4622343603311428, + "grad_norm": 2.5091891288757324, + "kl": 5.4296875, + "learning_rate": 6.858926952240925e-07, + "loss": 0.3761, + "num_tokens": 769187614.0, + "reward": 1.830078125, + "reward_std": 0.5374591946601868, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.841796875, + "rewards/format_reward/std": 0.36528825759887695, + "rewards/tag_count_reward/mean": 0.919921875, + "rewards/tag_count_reward/std": 0.20074127614498138, + "step": 1354 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1986.0, + "completions/mean_length": 809.958984375, + "completions/mean_terminated_length": 751.72802734375, + "completions/min_length": 180.0, + "completions/min_terminated_length": 180.0, + "epoch": 0.4625757446445336, + "grad_norm": 1.9437158107757568, + "kl": 6.796875, + "learning_rate": 6.853815201825016e-07, + "loss": 0.4782, + "num_tokens": 769676281.0, + "reward": 1.8330078125, + "reward_std": 0.5054324865341187, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.9169921875, + "rewards/tag_count_reward/std": 0.19831565022468567, + "step": 1355 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1930.0, + "completions/mean_length": 710.34375, + "completions/mean_terminated_length": 680.9740600585938, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "epoch": 0.46291712895792436, + "grad_norm": 2.3564820289611816, + "kl": 4.17578125, + "learning_rate": 6.848701529916047e-07, + "loss": 0.2856, + "num_tokens": 770120425.0, + "reward": 1.873046875, + "reward_std": 0.41014260053634644, + "rewards/accuracy_reward/mean": 0.03125, + "rewards/accuracy_reward/std": 0.17416280508041382, + "rewards/format_reward/mean": 0.892578125, + "rewards/format_reward/std": 0.30995169281959534, + "rewards/tag_count_reward/mean": 0.94921875, + "rewards/tag_count_reward/std": 0.15829749405384064, + "step": 1356 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1919.0, + "completions/mean_length": 736.501953125, + "completions/mean_terminated_length": 713.0357666015625, + "completions/min_length": 110.0, + "completions/min_terminated_length": 110.0, + "epoch": 0.4632585132713152, + "grad_norm": 2.157853841781616, + "kl": 4.48828125, + "learning_rate": 6.843585943771935e-07, + "loss": 0.3001, + "num_tokens": 770570266.0, + "reward": 1.859375, + "reward_std": 0.4438847303390503, + "rewards/accuracy_reward/mean": 0.025390625, + "rewards/accuracy_reward/std": 0.15746226906776428, + "rewards/format_reward/mean": 0.888671875, + "rewards/format_reward/std": 0.31484565138816833, + "rewards/tag_count_reward/mean": 0.9453125, + "rewards/tag_count_reward/std": 0.1631019562482834, + "step": 1357 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1932.0, + "completions/mean_length": 797.623046875, + "completions/mean_terminated_length": 749.43408203125, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "epoch": 0.463599897584706, + "grad_norm": 1.9284049272537231, + "kl": 6.4765625, + "learning_rate": 6.838468450653322e-07, + "loss": 0.3799, + "num_tokens": 771063209.0, + "reward": 1.830078125, + "reward_std": 0.597848653793335, + "rewards/accuracy_reward/mean": 0.109375, + "rewards/accuracy_reward/std": 0.31241437792778015, + "rewards/format_reward/mean": 0.810546875, + "rewards/format_reward/std": 0.3922513723373413, + "rewards/tag_count_reward/mean": 0.91015625, + "rewards/tag_count_reward/std": 0.20388682186603546, + "step": 1358 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2025.0, + "completions/mean_length": 721.576171875, + "completions/mean_terminated_length": 676.0222778320312, + "completions/min_length": 169.0, + "completions/min_terminated_length": 169.0, + "epoch": 0.4639412818980968, + "grad_norm": 2.0085532665252686, + "kl": 6.125, + "learning_rate": 6.833349057823553e-07, + "loss": 0.3542, + "num_tokens": 771518112.0, + "reward": 1.75, + "reward_std": 0.5615205764770508, + "rewards/accuracy_reward/mean": 0.03515625, + "rewards/accuracy_reward/std": 0.1843547374010086, + "rewards/format_reward/mean": 0.806640625, + "rewards/format_reward/std": 0.39531853795051575, + "rewards/tag_count_reward/mean": 0.908203125, + "rewards/tag_count_reward/std": 0.2095356285572052, + "step": 1359 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2010.0, + "completions/mean_length": 841.880859375, + "completions/mean_terminated_length": 766.8112182617188, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "epoch": 0.46428266621148756, + "grad_norm": 1.0249884128570557, + "kl": 6.2421875, + "learning_rate": 6.828227772548669e-07, + "loss": 0.4018, + "num_tokens": 772050259.0, + "reward": 1.7568359375, + "reward_std": 0.5212250351905823, + "rewards/accuracy_reward/mean": 0.025390625, + "rewards/accuracy_reward/std": 0.15746226906776428, + "rewards/format_reward/mean": 0.818359375, + "rewards/format_reward/std": 0.38592514395713806, + "rewards/tag_count_reward/mean": 0.9130859375, + "rewards/tag_count_reward/std": 0.20634421706199646, + "step": 1360 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1936.0, + "completions/mean_length": 770.390625, + "completions/mean_terminated_length": 723.8380737304688, + "completions/min_length": 161.0, + "completions/min_terminated_length": 161.0, + "epoch": 0.4646240505248784, + "grad_norm": 2.542614221572876, + "kl": 6.12109375, + "learning_rate": 6.823104602097398e-07, + "loss": 0.4169, + "num_tokens": 772525627.0, + "reward": 1.84619140625, + "reward_std": 0.578186571598053, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.833984375, + "rewards/format_reward/std": 0.3724585771560669, + "rewards/tag_count_reward/mean": 0.92626953125, + "rewards/tag_count_reward/std": 0.18913322687149048, + "step": 1361 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2022.0, + "completions/mean_length": 820.62890625, + "completions/mean_terminated_length": 749.6239624023438, + "completions/min_length": 5.0, + "completions/min_terminated_length": 5.0, + "epoch": 0.4649654348382692, + "grad_norm": 1.1544690132141113, + "kl": 7.65625, + "learning_rate": 6.817979553741143e-07, + "loss": 0.5048, + "num_tokens": 773029709.0, + "reward": 1.7685546875, + "reward_std": 0.5714544057846069, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.802734375, + "rewards/format_reward/std": 0.3983237147331238, + "rewards/tag_count_reward/mean": 0.9111328125, + "rewards/tag_count_reward/std": 0.21310555934906006, + "step": 1362 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1988.0, + "completions/mean_length": 771.203125, + "completions/mean_terminated_length": 732.66796875, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "epoch": 0.46530681915166, + "grad_norm": 0.9877282977104187, + "kl": 6.2109375, + "learning_rate": 6.812852634753974e-07, + "loss": 0.3745, + "num_tokens": 773505909.0, + "reward": 1.84130859375, + "reward_std": 0.5676465034484863, + "rewards/accuracy_reward/mean": 0.0927419364452362, + "rewards/accuracy_reward/std": 0.2903633117675781, + "rewards/format_reward/mean": 0.830078125, + "rewards/format_reward/std": 0.3759314715862274, + "rewards/tag_count_reward/mean": 0.92138671875, + "rewards/tag_count_reward/std": 0.19294461607933044, + "step": 1363 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 807.451171875, + "completions/mean_terminated_length": 754.3931274414062, + "completions/min_length": 129.0, + "completions/min_terminated_length": 129.0, + "epoch": 0.46564820346505076, + "grad_norm": 2.3786368370056152, + "kl": 7.734375, + "learning_rate": 6.807723852412613e-07, + "loss": 0.4793, + "num_tokens": 773999612.0, + "reward": 1.71826171875, + "reward_std": 0.5370117425918579, + "rewards/accuracy_reward/mean": 0.009765625, + "rewards/accuracy_reward/std": 0.09843364357948303, + "rewards/format_reward/mean": 0.802734375, + "rewards/format_reward/std": 0.3983237147331238, + "rewards/tag_count_reward/mean": 0.90576171875, + "rewards/tag_count_reward/std": 0.20874005556106567, + "step": 1364 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2016.0, + "completions/mean_length": 836.08203125, + "completions/mean_terminated_length": 794.4606323242188, + "completions/min_length": 186.0, + "completions/min_terminated_length": 186.0, + "epoch": 0.46598958777844157, + "grad_norm": 1.763398289680481, + "kl": 6.75, + "learning_rate": 6.802593213996431e-07, + "loss": 0.4171, + "num_tokens": 774510582.0, + "reward": 1.79736328125, + "reward_std": 0.5802618265151978, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.818359375, + "rewards/format_reward/std": 0.38592514395713806, + "rewards/tag_count_reward/mean": 0.91455078125, + "rewards/tag_count_reward/std": 0.20427954196929932, + "step": 1365 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1915.0, + "completions/mean_length": 784.43359375, + "completions/mean_terminated_length": 738.3927612304688, + "completions/min_length": 170.0, + "completions/min_terminated_length": 170.0, + "epoch": 0.4663309720918324, + "grad_norm": 15.666436195373535, + "kl": 5.6484375, + "learning_rate": 6.797460726787427e-07, + "loss": 0.3623, + "num_tokens": 774986852.0, + "reward": 1.81982421875, + "reward_std": 0.5481522679328918, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.92529296875, + "rewards/tag_count_reward/std": 0.19323164224624634, + "step": 1366 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 780.939453125, + "completions/mean_terminated_length": 732.1074829101562, + "completions/min_length": 120.0, + "completions/min_terminated_length": 120.0, + "epoch": 0.4666723564052232, + "grad_norm": 1.3910210132598877, + "kl": 5.87890625, + "learning_rate": 6.792326398070233e-07, + "loss": 0.3933, + "num_tokens": 775457109.0, + "reward": 1.8701171875, + "reward_std": 0.5559483170509338, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.9267578125, + "rewards/tag_count_reward/std": 0.19536417722702026, + "step": 1367 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2005.0, + "completions/mean_length": 794.640625, + "completions/mean_terminated_length": 761.9879760742188, + "completions/min_length": 165.0, + "completions/min_terminated_length": 165.0, + "epoch": 0.46701374071861396, + "grad_norm": 3.575126886367798, + "kl": 5.0859375, + "learning_rate": 6.787190235132085e-07, + "loss": 0.3598, + "num_tokens": 775936445.0, + "reward": 1.81201171875, + "reward_std": 0.47381865978240967, + "rewards/accuracy_reward/mean": 0.024193547666072845, + "rewards/accuracy_reward/std": 0.15380479395389557, + "rewards/format_reward/mean": 0.857421875, + "rewards/format_reward/std": 0.3499840497970581, + "rewards/tag_count_reward/mean": 0.93115234375, + "rewards/tag_count_reward/std": 0.18643119931221008, + "step": 1368 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2004.0, + "completions/mean_length": 764.12890625, + "completions/mean_terminated_length": 728.0361328125, + "completions/min_length": 158.0, + "completions/min_terminated_length": 158.0, + "epoch": 0.46735512503200477, + "grad_norm": 2.987510919570923, + "kl": 5.796875, + "learning_rate": 6.782052245262829e-07, + "loss": 0.3871, + "num_tokens": 776405071.0, + "reward": 1.80419921875, + "reward_std": 0.5042265057563782, + "rewards/accuracy_reward/mean": 0.030241934582591057, + "rewards/accuracy_reward/std": 0.1714252382516861, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.92919921875, + "rewards/tag_count_reward/std": 0.18635430932044983, + "step": 1369 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2001.0, + "completions/mean_length": 796.1015625, + "completions/mean_terminated_length": 745.2113647460938, + "completions/min_length": 198.0, + "completions/min_terminated_length": 198.0, + "epoch": 0.4676965093453956, + "grad_norm": 1.3319543600082397, + "kl": 6.6328125, + "learning_rate": 6.7769124357549e-07, + "loss": 0.4311, + "num_tokens": 776897299.0, + "reward": 1.83935546875, + "reward_std": 0.5210530757904053, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.92529296875, + "rewards/tag_count_reward/std": 0.19386357069015503, + "step": 1370 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 717.728515625, + "completions/mean_terminated_length": 672.04248046875, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "epoch": 0.4680378936587864, + "grad_norm": 3.551114320755005, + "kl": 7.1953125, + "learning_rate": 6.77177081390332e-07, + "loss": 0.4403, + "num_tokens": 777334968.0, + "reward": 1.818359375, + "reward_std": 0.5296976566314697, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.826171875, + "rewards/format_reward/std": 0.3793322443962097, + "rewards/tag_count_reward/mean": 0.919921875, + "rewards/tag_count_reward/std": 0.19642995297908783, + "step": 1371 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1938.0, + "completions/mean_length": 783.712890625, + "completions/mean_terminated_length": 750.7755737304688, + "completions/min_length": 175.0, + "completions/min_terminated_length": 175.0, + "epoch": 0.46837927797217715, + "grad_norm": 1.4473611116409302, + "kl": 6.6015625, + "learning_rate": 6.76662738700568e-07, + "loss": 0.4135, + "num_tokens": 777804229.0, + "reward": 1.90625, + "reward_std": 0.5176564455032349, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.873046875, + "rewards/format_reward/std": 0.33324605226516724, + "rewards/tag_count_reward/mean": 0.935546875, + "rewards/tag_count_reward/std": 0.18438583612442017, + "step": 1372 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 785.662109375, + "completions/mean_terminated_length": 726.288330078125, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "epoch": 0.46872066228556797, + "grad_norm": 3.8746228218078613, + "kl": 8.3359375, + "learning_rate": 6.761482162362134e-07, + "loss": 0.4935, + "num_tokens": 778278056.0, + "reward": 1.751953125, + "reward_std": 0.5673034191131592, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.802734375, + "rewards/format_reward/std": 0.3983237147331238, + "rewards/tag_count_reward/mean": 0.90625, + "rewards/tag_count_reward/std": 0.21558640897274017, + "step": 1373 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2010.0, + "completions/mean_length": 795.357421875, + "completions/mean_terminated_length": 741.7821044921875, + "completions/min_length": 193.0, + "completions/min_terminated_length": 193.0, + "epoch": 0.4690620465989588, + "grad_norm": 3.369460105895996, + "kl": 9.390625, + "learning_rate": 6.756335147275387e-07, + "loss": 0.6074, + "num_tokens": 778760831.0, + "reward": 1.80712890625, + "reward_std": 0.5608662366867065, + "rewards/accuracy_reward/mean": 0.06854838877916336, + "rewards/accuracy_reward/std": 0.25293973088264465, + "rewards/format_reward/mean": 0.828125, + "rewards/format_reward/std": 0.3776407241821289, + "rewards/tag_count_reward/mean": 0.91259765625, + "rewards/tag_count_reward/std": 0.2099587321281433, + "step": 1374 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2003.0, + "completions/mean_length": 825.08984375, + "completions/mean_terminated_length": 754.3429565429688, + "completions/min_length": 143.0, + "completions/min_terminated_length": 143.0, + "epoch": 0.4694034309123496, + "grad_norm": 1.8571772575378418, + "kl": 8.59375, + "learning_rate": 6.751186349050683e-07, + "loss": 0.5455, + "num_tokens": 779264509.0, + "reward": 1.76318359375, + "reward_std": 0.597456693649292, + "rewards/accuracy_reward/mean": 0.05443548411130905, + "rewards/accuracy_reward/std": 0.227104052901268, + "rewards/format_reward/mean": 0.802734375, + "rewards/format_reward/std": 0.3983237147331238, + "rewards/tag_count_reward/mean": 0.90771484375, + "rewards/tag_count_reward/std": 0.2084423005580902, + "step": 1375 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 801.888671875, + "completions/mean_terminated_length": 764.2796630859375, + "completions/min_length": 189.0, + "completions/min_terminated_length": 189.0, + "epoch": 0.46974481522574035, + "grad_norm": 1.4824538230895996, + "kl": 6.3984375, + "learning_rate": 6.746035774995805e-07, + "loss": 0.417, + "num_tokens": 779752532.0, + "reward": 1.83544921875, + "reward_std": 0.48997873067855835, + "rewards/accuracy_reward/mean": 0.052419353276491165, + "rewards/accuracy_reward/std": 0.22309619188308716, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.93115234375, + "rewards/tag_count_reward/std": 0.18445254862308502, + "step": 1376 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2024.0, + "completions/mean_length": 767.33984375, + "completions/mean_terminated_length": 728.6881103515625, + "completions/min_length": 78.0, + "completions/min_terminated_length": 78.0, + "epoch": 0.47008619953913117, + "grad_norm": 1.6078094244003296, + "kl": 5.5, + "learning_rate": 6.740883432421044e-07, + "loss": 0.3703, + "num_tokens": 780219154.0, + "reward": 1.84619140625, + "reward_std": 0.5149907469749451, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.92822265625, + "rewards/tag_count_reward/std": 0.18399609625339508, + "step": 1377 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1966.0, + "completions/mean_length": 856.470703125, + "completions/mean_terminated_length": 795.303955078125, + "completions/min_length": 118.0, + "completions/min_terminated_length": 118.0, + "epoch": 0.470427583852522, + "grad_norm": 2.1424074172973633, + "kl": 6.203125, + "learning_rate": 6.735729328639213e-07, + "loss": 0.406, + "num_tokens": 780738467.0, + "reward": 1.77685546875, + "reward_std": 0.5383716821670532, + "rewards/accuracy_reward/mean": 0.024193547666072845, + "rewards/accuracy_reward/std": 0.15380479395389557, + "rewards/format_reward/mean": 0.841796875, + "rewards/format_reward/std": 0.36528825759887695, + "rewards/tag_count_reward/mean": 0.91162109375, + "rewards/tag_count_reward/std": 0.2095487415790558, + "step": 1378 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 833.509765625, + "completions/mean_terminated_length": 776.386474609375, + "completions/min_length": 236.0, + "completions/min_terminated_length": 236.0, + "epoch": 0.4707689681659128, + "grad_norm": 2.3079988956451416, + "kl": 6.8125, + "learning_rate": 6.730573470965618e-07, + "loss": 0.4914, + "num_tokens": 781239752.0, + "reward": 1.77685546875, + "reward_std": 0.5395263433456421, + "rewards/accuracy_reward/mean": 0.03515625, + "rewards/accuracy_reward/std": 0.1843547374010086, + "rewards/format_reward/mean": 0.828125, + "rewards/format_reward/std": 0.3776407241821289, + "rewards/tag_count_reward/mean": 0.91357421875, + "rewards/tag_count_reward/std": 0.2032666653394699, + "step": 1379 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1948.0, + "completions/mean_length": 745.91015625, + "completions/mean_terminated_length": 701.1919555664062, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "epoch": 0.47111035247930355, + "grad_norm": 1.399194598197937, + "kl": 5.16015625, + "learning_rate": 6.725415866718055e-07, + "loss": 0.3408, + "num_tokens": 781711930.0, + "reward": 1.85107421875, + "reward_std": 0.47656652331352234, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.93310546875, + "rewards/tag_count_reward/std": 0.1851712167263031, + "step": 1380 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1958.0, + "completions/mean_length": 741.94140625, + "completions/mean_terminated_length": 677.708984375, + "completions/min_length": 187.0, + "completions/min_terminated_length": 187.0, + "epoch": 0.47145173679269436, + "grad_norm": 3.3459203243255615, + "kl": 6.53125, + "learning_rate": 6.720256523216802e-07, + "loss": 0.3746, + "num_tokens": 782176380.0, + "reward": 1.79150390625, + "reward_std": 0.5550722479820251, + "rewards/accuracy_reward/mean": 0.06451612710952759, + "rewards/accuracy_reward/std": 0.2459181249141693, + "rewards/format_reward/mean": 0.818359375, + "rewards/format_reward/std": 0.38592514395713806, + "rewards/tag_count_reward/mean": 0.91064453125, + "rewards/tag_count_reward/std": 0.20440112054347992, + "step": 1381 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2020.0, + "completions/mean_length": 893.115234375, + "completions/mean_terminated_length": 795.24365234375, + "completions/min_length": 231.0, + "completions/min_terminated_length": 231.0, + "epoch": 0.4717931211060852, + "grad_norm": 2.397712469100952, + "kl": 8.375, + "learning_rate": 6.715095447784602e-07, + "loss": 0.5589, + "num_tokens": 782708455.0, + "reward": 1.69482421875, + "reward_std": 0.5570254325866699, + "rewards/accuracy_reward/mean": 0.01171875, + "rewards/accuracy_reward/std": 0.10772226005792618, + "rewards/format_reward/mean": 0.794921875, + "rewards/format_reward/std": 0.4041535556316376, + "rewards/tag_count_reward/mean": 0.88818359375, + "rewards/tag_count_reward/std": 0.23107852041721344, + "step": 1382 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07421875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1972.0, + "completions/mean_length": 866.09765625, + "completions/mean_terminated_length": 771.345947265625, + "completions/min_length": 115.0, + "completions/min_terminated_length": 115.0, + "epoch": 0.472134505419476, + "grad_norm": 1.7249161005020142, + "kl": 8.0703125, + "learning_rate": 6.709932647746659e-07, + "loss": 0.5049, + "num_tokens": 783222281.0, + "reward": 1.70947265625, + "reward_std": 0.597592830657959, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.77734375, + "rewards/format_reward/std": 0.41643625497817993, + "rewards/tag_count_reward/mean": 0.88720703125, + "rewards/tag_count_reward/std": 0.22953926026821136, + "step": 1383 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2011.0, + "completions/mean_length": 775.4921875, + "completions/mean_terminated_length": 712.9097900390625, + "completions/min_length": 96.0, + "completions/min_terminated_length": 96.0, + "epoch": 0.47247588973286675, + "grad_norm": 2.245136260986328, + "kl": 7.265625, + "learning_rate": 6.704768130430619e-07, + "loss": 0.4825, + "num_tokens": 783694165.0, + "reward": 1.78857421875, + "reward_std": 0.5597871541976929, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.796875, + "rewards/format_reward/std": 0.4027182459831238, + "rewards/tag_count_reward/mean": 0.90576171875, + "rewards/tag_count_reward/std": 0.21164949238300323, + "step": 1384 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 783.619140625, + "completions/mean_terminated_length": 732.2214965820312, + "completions/min_length": 112.0, + "completions/min_terminated_length": 112.0, + "epoch": 0.47281727404625756, + "grad_norm": 2.1172027587890625, + "kl": 5.484375, + "learning_rate": 6.699601903166575e-07, + "loss": 0.3793, + "num_tokens": 784180482.0, + "reward": 1.83544921875, + "reward_std": 0.47278302907943726, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.93115234375, + "rewards/tag_count_reward/std": 0.1837882399559021, + "step": 1385 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.064453125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1858.0, + "completions/mean_length": 856.73046875, + "completions/mean_terminated_length": 774.6597290039062, + "completions/min_length": 184.0, + "completions/min_terminated_length": 184.0, + "epoch": 0.4731586583596484, + "grad_norm": 1.7951167821884155, + "kl": 7.1953125, + "learning_rate": 6.69443397328704e-07, + "loss": 0.4726, + "num_tokens": 784693864.0, + "reward": 1.83056640625, + "reward_std": 0.6076481342315674, + "rewards/accuracy_reward/mean": 0.111328125, + "rewards/accuracy_reward/std": 0.31484565138816833, + "rewards/format_reward/mean": 0.81640625, + "rewards/format_reward/std": 0.3875311613082886, + "rewards/tag_count_reward/mean": 0.90283203125, + "rewards/tag_count_reward/std": 0.2166205197572708, + "step": 1386 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07421875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1947.0, + "completions/mean_length": 850.3203125, + "completions/mean_terminated_length": 754.3037719726562, + "completions/min_length": 182.0, + "completions/min_terminated_length": 182.0, + "epoch": 0.4735000426730392, + "grad_norm": 1.806761384010315, + "kl": 7.796875, + "learning_rate": 6.689264348126944e-07, + "loss": 0.5443, + "num_tokens": 785207484.0, + "reward": 1.75732421875, + "reward_std": 0.5716904401779175, + "rewards/accuracy_reward/mean": 0.03515625, + "rewards/accuracy_reward/std": 0.1843547374010086, + "rewards/format_reward/mean": 0.82421875, + "rewards/format_reward/std": 0.3810062110424042, + "rewards/tag_count_reward/mean": 0.89794921875, + "rewards/tag_count_reward/std": 0.2391698956489563, + "step": 1387 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1954.0, + "completions/mean_length": 790.935546875, + "completions/mean_terminated_length": 712.695068359375, + "completions/min_length": 135.0, + "completions/min_terminated_length": 135.0, + "epoch": 0.47384142698642995, + "grad_norm": 1.3584413528442383, + "kl": 8.265625, + "learning_rate": 6.684093035023626e-07, + "loss": 0.5407, + "num_tokens": 785686187.0, + "reward": 1.81494140625, + "reward_std": 0.6129561066627502, + "rewards/accuracy_reward/mean": 0.119140625, + "rewards/accuracy_reward/std": 0.32427072525024414, + "rewards/format_reward/mean": 0.796875, + "rewards/format_reward/std": 0.4027182459831238, + "rewards/tag_count_reward/mean": 0.89892578125, + "rewards/tag_count_reward/std": 0.22319906949996948, + "step": 1388 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1997.0, + "completions/mean_length": 781.400390625, + "completions/mean_terminated_length": 729.91259765625, + "completions/min_length": 12.0, + "completions/min_terminated_length": 12.0, + "epoch": 0.47418281129982076, + "grad_norm": 2.051622152328491, + "kl": 7.609375, + "learning_rate": 6.678920041316818e-07, + "loss": 0.4913, + "num_tokens": 786168712.0, + "reward": 1.8271484375, + "reward_std": 0.5685397386550903, + "rewards/accuracy_reward/mean": 0.07459677755832672, + "rewards/accuracy_reward/std": 0.263004869222641, + "rewards/format_reward/mean": 0.83203125, + "rewards/format_reward/std": 0.374204158782959, + "rewards/tag_count_reward/mean": 0.9228515625, + "rewards/tag_count_reward/std": 0.19635941088199615, + "step": 1389 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.060546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2015.0, + "completions/mean_length": 810.513671875, + "completions/mean_terminated_length": 730.7588500976562, + "completions/min_length": 204.0, + "completions/min_terminated_length": 204.0, + "epoch": 0.4745241956132116, + "grad_norm": 3.496229648590088, + "kl": 8.8046875, + "learning_rate": 6.67374537434864e-07, + "loss": 0.5496, + "num_tokens": 786655439.0, + "reward": 1.77734375, + "reward_std": 0.5651187896728516, + "rewards/accuracy_reward/mean": 0.06451612710952759, + "rewards/accuracy_reward/std": 0.2459181249141693, + "rewards/format_reward/mean": 0.8125, + "rewards/format_reward/std": 0.39069411158561707, + "rewards/tag_count_reward/mean": 0.90234375, + "rewards/tag_count_reward/std": 0.2211524099111557, + "step": 1390 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2014.0, + "completions/mean_length": 844.98046875, + "completions/mean_terminated_length": 778.00830078125, + "completions/min_length": 209.0, + "completions/min_terminated_length": 209.0, + "epoch": 0.4748655799266024, + "grad_norm": 1.6232308149337769, + "kl": 8.71875, + "learning_rate": 6.668569041463582e-07, + "loss": 0.5677, + "num_tokens": 787171461.0, + "reward": 1.6982421875, + "reward_std": 0.5887287855148315, + "rewards/accuracy_reward/mean": 0.029296875, + "rewards/accuracy_reward/std": 0.16880230605602264, + "rewards/format_reward/mean": 0.779296875, + "rewards/format_reward/std": 0.4151262938976288, + "rewards/tag_count_reward/mean": 0.8896484375, + "rewards/tag_count_reward/std": 0.23257318139076233, + "step": 1391 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 829.41015625, + "completions/mean_terminated_length": 764.2180786132812, + "completions/min_length": 123.0, + "completions/min_terminated_length": 123.0, + "epoch": 0.47520696423999315, + "grad_norm": 1.2676295042037964, + "kl": 6.96875, + "learning_rate": 6.663391050008505e-07, + "loss": 0.476, + "num_tokens": 787675415.0, + "reward": 1.81396484375, + "reward_std": 0.504416823387146, + "rewards/accuracy_reward/mean": 0.03515625, + "rewards/accuracy_reward/std": 0.1843547374010086, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.92333984375, + "rewards/tag_count_reward/std": 0.19436074793338776, + "step": 1392 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2012.0, + "completions/mean_length": 792.07421875, + "completions/mean_terminated_length": 741.0203247070312, + "completions/min_length": 71.0, + "completions/min_terminated_length": 71.0, + "epoch": 0.47554834855338396, + "grad_norm": 2.010774850845337, + "kl": 7.109375, + "learning_rate": 6.658211407332619e-07, + "loss": 0.4803, + "num_tokens": 788162637.0, + "reward": 1.8095703125, + "reward_std": 0.5974607467651367, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.828125, + "rewards/format_reward/std": 0.3776407241821289, + "rewards/tag_count_reward/mean": 0.9130859375, + "rewards/tag_count_reward/std": 0.211032897233963, + "step": 1393 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1973.0, + "completions/mean_length": 790.86328125, + "completions/mean_terminated_length": 726.3285522460938, + "completions/min_length": 107.0, + "completions/min_terminated_length": 107.0, + "epoch": 0.4758897328667748, + "grad_norm": 1.3909481763839722, + "kl": 8.2578125, + "learning_rate": 6.65303012078748e-07, + "loss": 0.5387, + "num_tokens": 788641143.0, + "reward": 1.78564453125, + "reward_std": 0.6270242929458618, + "rewards/accuracy_reward/mean": 0.087890625, + "rewards/accuracy_reward/std": 0.2834126651287079, + "rewards/format_reward/mean": 0.80078125, + "rewards/format_reward/std": 0.39980348944664, + "rewards/tag_count_reward/mean": 0.89697265625, + "rewards/tag_count_reward/std": 0.22773799300193787, + "step": 1394 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1699.0, + "completions/mean_length": 804.697265625, + "completions/mean_terminated_length": 751.5214233398438, + "completions/min_length": 177.0, + "completions/min_terminated_length": 177.0, + "epoch": 0.4762311171801656, + "grad_norm": 2.7253928184509277, + "kl": 5.2421875, + "learning_rate": 6.647847197726978e-07, + "loss": 0.3513, + "num_tokens": 789132396.0, + "reward": 1.876953125, + "reward_std": 0.494552344083786, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.9375, + "rewards/tag_count_reward/std": 0.17901119589805603, + "step": 1395 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2014.0, + "completions/mean_length": 865.7578125, + "completions/mean_terminated_length": 805.0678100585938, + "completions/min_length": 194.0, + "completions/min_terminated_length": 194.0, + "epoch": 0.47657250149355634, + "grad_norm": 2.511202335357666, + "kl": 6.2890625, + "learning_rate": 6.642662645507322e-07, + "loss": 0.4536, + "num_tokens": 789652256.0, + "reward": 1.83154296875, + "reward_std": 0.5061460733413696, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.92919921875, + "rewards/tag_count_reward/std": 0.18896137177944183, + "step": 1396 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2012.0, + "completions/mean_length": 856.08984375, + "completions/mean_terminated_length": 797.4712524414062, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "epoch": 0.47691388580694716, + "grad_norm": 3.641901731491089, + "kl": 8.0546875, + "learning_rate": 6.637476471487036e-07, + "loss": 0.4617, + "num_tokens": 790164814.0, + "reward": 1.7822265625, + "reward_std": 0.5402143597602844, + "rewards/accuracy_reward/mean": 0.05443548411130905, + "rewards/accuracy_reward/std": 0.227104052901268, + "rewards/format_reward/mean": 0.814453125, + "rewards/format_reward/std": 0.38912075757980347, + "rewards/tag_count_reward/mean": 0.9150390625, + "rewards/tag_count_reward/std": 0.20055793225765228, + "step": 1397 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1811.0, + "completions/mean_length": 817.111328125, + "completions/mean_terminated_length": 751.2612915039062, + "completions/min_length": 20.0, + "completions/min_terminated_length": 20.0, + "epoch": 0.47725527012033797, + "grad_norm": 2.3625988960266113, + "kl": 7.6953125, + "learning_rate": 6.632288683026946e-07, + "loss": 0.4697, + "num_tokens": 790654695.0, + "reward": 1.8330078125, + "reward_std": 0.5581883788108826, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.833984375, + "rewards/format_reward/std": 0.3724585771560669, + "rewards/tag_count_reward/mean": 0.9169921875, + "rewards/tag_count_reward/std": 0.2031896710395813, + "step": 1398 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2033.0, + "completions/mean_length": 853.94921875, + "completions/mean_terminated_length": 792.6530151367188, + "completions/min_length": 32.0, + "completions/min_terminated_length": 32.0, + "epoch": 0.4775966544337288, + "grad_norm": 3.0895321369171143, + "kl": 8.1875, + "learning_rate": 6.62709928749017e-07, + "loss": 0.4903, + "num_tokens": 791178909.0, + "reward": 1.70703125, + "reward_std": 0.5963179469108582, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.771484375, + "rewards/format_reward/std": 0.4202871024608612, + "rewards/tag_count_reward/mean": 0.88671875, + "rewards/tag_count_reward/std": 0.22903135418891907, + "step": 1399 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1985.0, + "completions/mean_length": 798.802734375, + "completions/mean_terminated_length": 755.9010620117188, + "completions/min_length": 155.0, + "completions/min_terminated_length": 155.0, + "epoch": 0.47793803874711954, + "grad_norm": 1.0735238790512085, + "kl": 5.6328125, + "learning_rate": 6.621908292242104e-07, + "loss": 0.33, + "num_tokens": 791664888.0, + "reward": 1.7802734375, + "reward_std": 0.5403667688369751, + "rewards/accuracy_reward/mean": 0.05443548411130905, + "rewards/accuracy_reward/std": 0.227104052901268, + "rewards/format_reward/mean": 0.810546875, + "rewards/format_reward/std": 0.3922513723373413, + "rewards/tag_count_reward/mean": 0.9169921875, + "rewards/tag_count_reward/std": 0.19458001852035522, + "step": 1400 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1981.0, + "completions/mean_length": 818.6015625, + "completions/mean_terminated_length": 763.404052734375, + "completions/min_length": 206.0, + "completions/min_terminated_length": 206.0, + "epoch": 0.47827942306051036, + "grad_norm": 1.592006802558899, + "kl": 7.0078125, + "learning_rate": 6.616715704650418e-07, + "loss": 0.4583, + "num_tokens": 792159836.0, + "reward": 1.79052734375, + "reward_std": 0.5385003089904785, + "rewards/accuracy_reward/mean": 0.029296875, + "rewards/accuracy_reward/std": 0.16880230605602264, + "rewards/format_reward/mean": 0.841796875, + "rewards/format_reward/std": 0.36528825759887695, + "rewards/tag_count_reward/mean": 0.91943359375, + "rewards/tag_count_reward/std": 0.20327135920524597, + "step": 1401 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2028.0, + "completions/mean_length": 807.796875, + "completions/mean_terminated_length": 775.4869995117188, + "completions/min_length": 165.0, + "completions/min_terminated_length": 165.0, + "epoch": 0.47862080737390117, + "grad_norm": 1.1120721101760864, + "kl": 5.20703125, + "learning_rate": 6.611521532085038e-07, + "loss": 0.3211, + "num_tokens": 792656564.0, + "reward": 1.80224609375, + "reward_std": 0.5459111928939819, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.8359375, + "rewards/format_reward/std": 0.37069445848464966, + "rewards/tag_count_reward/mean": 0.91943359375, + "rewards/tag_count_reward/std": 0.19778190553188324, + "step": 1402 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1977.0, + "completions/mean_length": 827.939453125, + "completions/mean_terminated_length": 780.9188232421875, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "epoch": 0.478962191687292, + "grad_norm": 0.9376634955406189, + "kl": 6.22265625, + "learning_rate": 6.606325781918144e-07, + "loss": 0.3956, + "num_tokens": 793160389.0, + "reward": 1.73974609375, + "reward_std": 0.5454986691474915, + "rewards/accuracy_reward/mean": 0.02217741869390011, + "rewards/accuracy_reward/std": 0.14740893244743347, + "rewards/format_reward/mean": 0.806640625, + "rewards/format_reward/std": 0.39531853795051575, + "rewards/tag_count_reward/mean": 0.91162109375, + "rewards/tag_count_reward/std": 0.20302677154541016, + "step": 1403 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2031.0, + "completions/mean_length": 850.501953125, + "completions/mean_terminated_length": 801.8231201171875, + "completions/min_length": 115.0, + "completions/min_terminated_length": 115.0, + "epoch": 0.4793035760006828, + "grad_norm": 1.7362650632858276, + "kl": 6.296875, + "learning_rate": 6.601128461524152e-07, + "loss": 0.4108, + "num_tokens": 793665142.0, + "reward": 1.76904296875, + "reward_std": 0.5725255608558655, + "rewards/accuracy_reward/mean": 0.04435483738780022, + "rewards/accuracy_reward/std": 0.2060900777578354, + "rewards/format_reward/mean": 0.810546875, + "rewards/format_reward/std": 0.3922513723373413, + "rewards/tag_count_reward/mean": 0.91552734375, + "rewards/tag_count_reward/std": 0.19551756978034973, + "step": 1404 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1996.0, + "completions/mean_length": 827.376953125, + "completions/mean_terminated_length": 762.0761108398438, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "epoch": 0.47964496031407355, + "grad_norm": 1.6607929468154907, + "kl": 7.4375, + "learning_rate": 6.595929578279708e-07, + "loss": 0.5082, + "num_tokens": 794168279.0, + "reward": 1.771484375, + "reward_std": 0.5720074772834778, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.8046875, + "rewards/format_reward/std": 0.3968288004398346, + "rewards/tag_count_reward/mean": 0.91015625, + "rewards/tag_count_reward/std": 0.20979996025562286, + "step": 1405 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 766.435546875, + "completions/mean_terminated_length": 719.7388916015625, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "epoch": 0.47998634462746437, + "grad_norm": 3.086946725845337, + "kl": 7.2265625, + "learning_rate": 6.590729139563675e-07, + "loss": 0.4113, + "num_tokens": 794639558.0, + "reward": 1.77392578125, + "reward_std": 0.6171959042549133, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.787109375, + "rewards/format_reward/std": 0.409751296043396, + "rewards/tag_count_reward/mean": 0.89697265625, + "rewards/tag_count_reward/std": 0.22064577043056488, + "step": 1406 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1947.0, + "completions/mean_length": 781.287109375, + "completions/mean_terminated_length": 737.7838745117188, + "completions/min_length": 50.0, + "completions/min_terminated_length": 50.0, + "epoch": 0.4803277289408552, + "grad_norm": 1.5814425945281982, + "kl": 7.203125, + "learning_rate": 6.585527152757128e-07, + "loss": 0.4377, + "num_tokens": 795114585.0, + "reward": 1.81787109375, + "reward_std": 0.5788640379905701, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.82421875, + "rewards/format_reward/std": 0.3810062110424042, + "rewards/tag_count_reward/mean": 0.91357421875, + "rewards/tag_count_reward/std": 0.20743593573570251, + "step": 1407 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1949.0, + "completions/mean_length": 725.12109375, + "completions/mean_terminated_length": 676.9190673828125, + "completions/min_length": 52.0, + "completions/min_terminated_length": 52.0, + "epoch": 0.480669113254246, + "grad_norm": 2.2530622482299805, + "kl": 6.734375, + "learning_rate": 6.580323625243332e-07, + "loss": 0.4232, + "num_tokens": 795561687.0, + "reward": 1.81689453125, + "reward_std": 0.5578827261924744, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.841796875, + "rewards/format_reward/std": 0.36528825759887695, + "rewards/tag_count_reward/mean": 0.92236328125, + "rewards/tag_count_reward/std": 0.20260746777057648, + "step": 1408 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1962.0, + "completions/mean_length": 718.115234375, + "completions/mean_terminated_length": 686.1980590820312, + "completions/min_length": 190.0, + "completions/min_terminated_length": 190.0, + "epoch": 0.48101049756763675, + "grad_norm": 2.0103352069854736, + "kl": 5.1796875, + "learning_rate": 6.575118564407742e-07, + "loss": 0.3535, + "num_tokens": 796008514.0, + "reward": 1.9111328125, + "reward_std": 0.49185386300086975, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.873046875, + "rewards/format_reward/std": 0.33324605226516724, + "rewards/tag_count_reward/mean": 0.9462890625, + "rewards/tag_count_reward/std": 0.15418460965156555, + "step": 1409 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 759.279296875, + "completions/mean_terminated_length": 725.7054443359375, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "epoch": 0.48135188188102757, + "grad_norm": 2.951759099960327, + "kl": 6.1328125, + "learning_rate": 6.569911977637994e-07, + "loss": 0.4127, + "num_tokens": 796480241.0, + "reward": 1.82080078125, + "reward_std": 0.5672469139099121, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.92431640625, + "rewards/tag_count_reward/std": 0.20031657814979553, + "step": 1410 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 804.99609375, + "completions/mean_terminated_length": 749.187744140625, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "epoch": 0.4816932661944184, + "grad_norm": 2.1062204837799072, + "kl": 7.015625, + "learning_rate": 6.564703872323883e-07, + "loss": 0.4235, + "num_tokens": 796975519.0, + "reward": 1.806640625, + "reward_std": 0.5600014925003052, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.81640625, + "rewards/format_reward/std": 0.3875311613082886, + "rewards/tag_count_reward/mean": 0.919921875, + "rewards/tag_count_reward/std": 0.19951897859573364, + "step": 1411 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1971.0, + "completions/mean_length": 775.48828125, + "completions/mean_terminated_length": 739.71484375, + "completions/min_length": 143.0, + "completions/min_terminated_length": 143.0, + "epoch": 0.4820346505078092, + "grad_norm": 1.2882215976715088, + "kl": 5.541015625, + "learning_rate": 6.559494255857362e-07, + "loss": 0.3697, + "num_tokens": 797452313.0, + "reward": 1.8349609375, + "reward_std": 0.4919002652168274, + "rewards/accuracy_reward/mean": 0.037109375, + "rewards/accuracy_reward/std": 0.18921469151973724, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.9326171875, + "rewards/tag_count_reward/std": 0.185324028134346, + "step": 1412 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1981.0, + "completions/mean_length": 717.904296875, + "completions/mean_terminated_length": 694.1053466796875, + "completions/min_length": 75.0, + "completions/min_terminated_length": 75.0, + "epoch": 0.48237603482119995, + "grad_norm": 2.3797285556793213, + "kl": 5.15625, + "learning_rate": 6.554283135632529e-07, + "loss": 0.3489, + "num_tokens": 797892824.0, + "reward": 1.87841796875, + "reward_std": 0.5279759764671326, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.86328125, + "rewards/format_reward/std": 0.3438861668109894, + "rewards/tag_count_reward/mean": 0.93701171875, + "rewards/tag_count_reward/std": 0.17642973363399506, + "step": 1413 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2008.0, + "completions/mean_length": 779.306640625, + "completions/mean_terminated_length": 738.3810424804688, + "completions/min_length": 165.0, + "completions/min_terminated_length": 165.0, + "epoch": 0.48271741913459076, + "grad_norm": 1.7999489307403564, + "kl": 6.3046875, + "learning_rate": 6.549070519045615e-07, + "loss": 0.4473, + "num_tokens": 798368725.0, + "reward": 1.8359375, + "reward_std": 0.4930545687675476, + "rewards/accuracy_reward/mean": 0.03125, + "rewards/accuracy_reward/std": 0.17416280508041382, + "rewards/format_reward/mean": 0.8671875, + "rewards/format_reward/std": 0.33970388770103455, + "rewards/tag_count_reward/mean": 0.9375, + "rewards/tag_count_reward/std": 0.17969314754009247, + "step": 1414 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1748.0, + "completions/mean_length": 686.099609375, + "completions/mean_terminated_length": 650.6192626953125, + "completions/min_length": 144.0, + "completions/min_terminated_length": 144.0, + "epoch": 0.4830588034479816, + "grad_norm": 1.749036431312561, + "kl": 5.484375, + "learning_rate": 6.543856413494979e-07, + "loss": 0.3711, + "num_tokens": 798794648.0, + "reward": 1.91064453125, + "reward_std": 0.4636210799217224, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.89453125, + "rewards/format_reward/std": 0.3074568510055542, + "rewards/tag_count_reward/mean": 0.94970703125, + "rewards/tag_count_reward/std": 0.1678251326084137, + "step": 1415 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1960.0, + "completions/mean_length": 725.783203125, + "completions/mean_terminated_length": 694.050048828125, + "completions/min_length": 69.0, + "completions/min_terminated_length": 69.0, + "epoch": 0.4834001877613724, + "grad_norm": 1.747126817703247, + "kl": 4.9921875, + "learning_rate": 6.538640826381086e-07, + "loss": 0.3184, + "num_tokens": 799245401.0, + "reward": 1.8642578125, + "reward_std": 0.4920358955860138, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.9365234375, + "rewards/tag_count_reward/std": 0.16953378915786743, + "step": 1416 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1951.0, + "completions/mean_length": 717.318359375, + "completions/mean_terminated_length": 682.6513061523438, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "epoch": 0.48374157207476315, + "grad_norm": 1.6827846765518188, + "kl": 6.3671875, + "learning_rate": 6.533423765106509e-07, + "loss": 0.3959, + "num_tokens": 799687372.0, + "reward": 1.81005859375, + "reward_std": 0.5226245522499084, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.830078125, + "rewards/format_reward/std": 0.3759314715862274, + "rewards/tag_count_reward/mean": 0.92724609375, + "rewards/tag_count_reward/std": 0.1888652741909027, + "step": 1417 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2007.0, + "completions/mean_length": 671.201171875, + "completions/mean_terminated_length": 654.8755493164062, + "completions/min_length": 151.0, + "completions/min_terminated_length": 151.0, + "epoch": 0.48408295638815396, + "grad_norm": 2.371227264404297, + "kl": 6.671875, + "learning_rate": 6.528205237075916e-07, + "loss": 0.382, + "num_tokens": 800112291.0, + "reward": 1.85986328125, + "reward_std": 0.532575249671936, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.93408203125, + "rewards/tag_count_reward/std": 0.17183120548725128, + "step": 1418 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1898.0, + "completions/mean_length": 730.56640625, + "completions/mean_terminated_length": 696.2445068359375, + "completions/min_length": 163.0, + "completions/min_terminated_length": 163.0, + "epoch": 0.4844243407015448, + "grad_norm": 3.0626814365386963, + "kl": 6.453125, + "learning_rate": 6.522985249696049e-07, + "loss": 0.3758, + "num_tokens": 800560037.0, + "reward": 1.892578125, + "reward_std": 0.5158536434173584, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.935546875, + "rewards/tag_count_reward/std": 0.1876731961965561, + "step": 1419 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2031.0, + "completions/mean_length": 754.92578125, + "completions/mean_terminated_length": 710.5172119140625, + "completions/min_length": 151.0, + "completions/min_terminated_length": 151.0, + "epoch": 0.4847657250149356, + "grad_norm": 3.68796706199646, + "kl": 8.90625, + "learning_rate": 6.517763810375727e-07, + "loss": 0.5475, + "num_tokens": 801022831.0, + "reward": 1.84814453125, + "reward_std": 0.5662318468093872, + "rewards/accuracy_reward/mean": 0.07661290466785431, + "rewards/accuracy_reward/std": 0.2662447690963745, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.92822265625, + "rewards/tag_count_reward/std": 0.1911684274673462, + "step": 1420 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1918.0, + "completions/mean_length": 709.62109375, + "completions/mean_terminated_length": 677.5000610351562, + "completions/min_length": 72.0, + "completions/min_terminated_length": 72.0, + "epoch": 0.48510710932832635, + "grad_norm": 1.6579664945602417, + "kl": 6.4453125, + "learning_rate": 6.512540926525828e-07, + "loss": 0.3901, + "num_tokens": 801452461.0, + "reward": 1.92333984375, + "reward_std": 0.5349385142326355, + "rewards/accuracy_reward/mean": 0.109375, + "rewards/accuracy_reward/std": 0.31241437792778015, + "rewards/format_reward/mean": 0.876953125, + "rewards/format_reward/std": 0.32881227135658264, + "rewards/tag_count_reward/mean": 0.93701171875, + "rewards/tag_count_reward/std": 0.18389739096164703, + "step": 1421 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1980.0, + "completions/mean_length": 756.48828125, + "completions/mean_terminated_length": 720.1807250976562, + "completions/min_length": 233.0, + "completions/min_terminated_length": 233.0, + "epoch": 0.48544849364171716, + "grad_norm": 1.7433476448059082, + "kl": 6.7890625, + "learning_rate": 6.507316605559281e-07, + "loss": 0.4027, + "num_tokens": 801916567.0, + "reward": 1.82568359375, + "reward_std": 0.5088048577308655, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.849609375, + "rewards/format_reward/std": 0.35780346393585205, + "rewards/tag_count_reward/mean": 0.93115234375, + "rewards/tag_count_reward/std": 0.18511444330215454, + "step": 1422 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2010.0, + "completions/mean_length": 739.330078125, + "completions/mean_terminated_length": 715.9144897460938, + "completions/min_length": 192.0, + "completions/min_terminated_length": 192.0, + "epoch": 0.485789877955108, + "grad_norm": 3.603640079498291, + "kl": 4.7109375, + "learning_rate": 6.502090854891051e-07, + "loss": 0.335, + "num_tokens": 802367072.0, + "reward": 1.931640625, + "reward_std": 0.48461073637008667, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.88671875, + "rewards/format_reward/std": 0.3172462284564972, + "rewards/tag_count_reward/mean": 0.953125, + "rewards/tag_count_reward/std": 0.153242826461792, + "step": 1423 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 789.9375, + "completions/mean_terminated_length": 754.5702514648438, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.4861312622684988, + "grad_norm": 1.4389578104019165, + "kl": 6.1171875, + "learning_rate": 6.496863681938138e-07, + "loss": 0.3802, + "num_tokens": 802846896.0, + "reward": 1.7919921875, + "reward_std": 0.535548985004425, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.81640625, + "rewards/format_reward/std": 0.3875311613082886, + "rewards/tag_count_reward/mean": 0.9208984375, + "rewards/tag_count_reward/std": 0.19179034233093262, + "step": 1424 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1757.0, + "completions/mean_length": 773.16015625, + "completions/mean_terminated_length": 734.68408203125, + "completions/min_length": 124.0, + "completions/min_terminated_length": 124.0, + "epoch": 0.48647264658188955, + "grad_norm": 1.6272501945495605, + "kl": 5.1328125, + "learning_rate": 6.491635094119558e-07, + "loss": 0.3192, + "num_tokens": 803316226.0, + "reward": 1.904296875, + "reward_std": 0.5398890972137451, + "rewards/accuracy_reward/mean": 0.125, + "rewards/accuracy_reward/std": 0.3310528099536896, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.931640625, + "rewards/tag_count_reward/std": 0.1823011338710785, + "step": 1425 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 775.666015625, + "completions/mean_terminated_length": 737.2655639648438, + "completions/min_length": 167.0, + "completions/min_terminated_length": 167.0, + "epoch": 0.48681403089528036, + "grad_norm": 1.0159920454025269, + "kl": 6.25, + "learning_rate": 6.486405098856333e-07, + "loss": 0.3864, + "num_tokens": 803787991.0, + "reward": 1.79345703125, + "reward_std": 0.531171441078186, + "rewards/accuracy_reward/mean": 0.033203125, + "rewards/accuracy_reward/std": 0.17934183776378632, + "rewards/format_reward/mean": 0.8359375, + "rewards/format_reward/std": 0.37069445848464966, + "rewards/tag_count_reward/mean": 0.92431640625, + "rewards/tag_count_reward/std": 0.19348366558551788, + "step": 1426 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1935.0, + "completions/mean_length": 740.220703125, + "completions/mean_terminated_length": 706.1503295898438, + "completions/min_length": 145.0, + "completions/min_terminated_length": 145.0, + "epoch": 0.48715541520867117, + "grad_norm": 0.8419586420059204, + "kl": 6.4765625, + "learning_rate": 6.481173703571487e-07, + "loss": 0.4009, + "num_tokens": 804246792.0, + "reward": 1.8125, + "reward_std": 0.5184415578842163, + "rewards/accuracy_reward/mean": 0.03515625, + "rewards/accuracy_reward/std": 0.1843547374010086, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.93359375, + "rewards/tag_count_reward/std": 0.17828376591205597, + "step": 1427 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2033.0, + "completions/mean_length": 835.873046875, + "completions/mean_terminated_length": 789.158203125, + "completions/min_length": 72.0, + "completions/min_terminated_length": 72.0, + "epoch": 0.487496799522062, + "grad_norm": 2.0123610496520996, + "kl": 6.5625, + "learning_rate": 6.475940915690028e-07, + "loss": 0.3536, + "num_tokens": 804751847.0, + "reward": 1.79638671875, + "reward_std": 0.5437809824943542, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.8125, + "rewards/format_reward/std": 0.39069411158561707, + "rewards/tag_count_reward/mean": 0.92724609375, + "rewards/tag_count_reward/std": 0.1927117109298706, + "step": 1428 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1971.0, + "completions/mean_length": 807.216796875, + "completions/mean_terminated_length": 764.6040649414062, + "completions/min_length": 146.0, + "completions/min_terminated_length": 146.0, + "epoch": 0.48783818383545274, + "grad_norm": 3.6934638023376465, + "kl": 7.0625, + "learning_rate": 6.470706742638942e-07, + "loss": 0.4154, + "num_tokens": 805246662.0, + "reward": 1.78173828125, + "reward_std": 0.5377598404884338, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.806640625, + "rewards/format_reward/std": 0.39531853795051575, + "rewards/tag_count_reward/mean": 0.91650390625, + "rewards/tag_count_reward/std": 0.19903406500816345, + "step": 1429 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 843.90625, + "completions/mean_terminated_length": 784.6884765625, + "completions/min_length": 142.0, + "completions/min_terminated_length": 142.0, + "epoch": 0.48817956814884356, + "grad_norm": 3.9663517475128174, + "kl": 8.9609375, + "learning_rate": 6.465471191847177e-07, + "loss": 0.5069, + "num_tokens": 805758886.0, + "reward": 1.7236328125, + "reward_std": 0.6423845291137695, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.765625, + "rewards/format_reward/std": 0.42402184009552, + "rewards/tag_count_reward/mean": 0.8935546875, + "rewards/tag_count_reward/std": 0.22588695585727692, + "step": 1430 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 840.4375, + "completions/mean_terminated_length": 796.437255859375, + "completions/min_length": 150.0, + "completions/min_terminated_length": 150.0, + "epoch": 0.48852095246223437, + "grad_norm": 0.9592890739440918, + "kl": 6.546875, + "learning_rate": 6.460234270745645e-07, + "loss": 0.3858, + "num_tokens": 806264070.0, + "reward": 1.84912109375, + "reward_std": 0.5959770679473877, + "rewards/accuracy_reward/mean": 0.1015625, + "rewards/accuracy_reward/std": 0.30236753821372986, + "rewards/format_reward/mean": 0.82421875, + "rewards/format_reward/std": 0.3810062110424042, + "rewards/tag_count_reward/mean": 0.92333984375, + "rewards/tag_count_reward/std": 0.19561529159545898, + "step": 1431 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1957.0, + "completions/mean_length": 857.744140625, + "completions/mean_terminated_length": 796.6427612304688, + "completions/min_length": 121.0, + "completions/min_terminated_length": 121.0, + "epoch": 0.4888623367756252, + "grad_norm": 1.1903507709503174, + "kl": 7.109375, + "learning_rate": 6.454995986767193e-07, + "loss": 0.3926, + "num_tokens": 806788211.0, + "reward": 1.70947265625, + "reward_std": 0.5695422887802124, + "rewards/accuracy_reward/mean": 0.029296875, + "rewards/accuracy_reward/std": 0.16880230605602264, + "rewards/format_reward/mean": 0.78125, + "rewards/format_reward/std": 0.41380295157432556, + "rewards/tag_count_reward/mean": 0.89892578125, + "rewards/tag_count_reward/std": 0.22319906949996948, + "step": 1432 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1953.0, + "completions/mean_length": 853.853515625, + "completions/mean_terminated_length": 802.7800903320312, + "completions/min_length": 199.0, + "completions/min_terminated_length": 199.0, + "epoch": 0.48920372108901594, + "grad_norm": 1.5274327993392944, + "kl": 5.78125, + "learning_rate": 6.44975634734661e-07, + "loss": 0.3492, + "num_tokens": 807299704.0, + "reward": 1.85107421875, + "reward_std": 0.6022671461105347, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.826171875, + "rewards/format_reward/std": 0.3793322443962097, + "rewards/tag_count_reward/mean": 0.91943359375, + "rewards/tag_count_reward/std": 0.19277119636535645, + "step": 1433 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2033.0, + "completions/mean_length": 851.294921875, + "completions/mean_terminated_length": 812.6915283203125, + "completions/min_length": 89.0, + "completions/min_terminated_length": 89.0, + "epoch": 0.48954510540240675, + "grad_norm": 1.284857153892517, + "kl": 5.73046875, + "learning_rate": 6.444515359920605e-07, + "loss": 0.3554, + "num_tokens": 807812031.0, + "reward": 1.76708984375, + "reward_std": 0.5164639949798584, + "rewards/accuracy_reward/mean": 0.038306452333927155, + "rewards/accuracy_reward/std": 0.19212883710861206, + "rewards/format_reward/mean": 0.8125, + "rewards/format_reward/std": 0.39069411158561707, + "rewards/tag_count_reward/mean": 0.91748046875, + "rewards/tag_count_reward/std": 0.19510170817375183, + "step": 1434 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2012.0, + "completions/mean_length": 874.740234375, + "completions/mean_terminated_length": 824.5601196289062, + "completions/min_length": 125.0, + "completions/min_terminated_length": 125.0, + "epoch": 0.48988648971579757, + "grad_norm": 1.281517505645752, + "kl": 6.0, + "learning_rate": 6.439273031927801e-07, + "loss": 0.3561, + "num_tokens": 808335802.0, + "reward": 1.80078125, + "reward_std": 0.5718402862548828, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.8046875, + "rewards/format_reward/std": 0.3968288004398346, + "rewards/tag_count_reward/mean": 0.912109375, + "rewards/tag_count_reward/std": 0.20473802089691162, + "step": 1435 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2025.0, + "completions/mean_length": 910.60546875, + "completions/mean_terminated_length": 861.9592895507812, + "completions/min_length": 164.0, + "completions/min_terminated_length": 164.0, + "epoch": 0.4902278740291884, + "grad_norm": 1.5556361675262451, + "kl": 5.3046875, + "learning_rate": 6.434029370808722e-07, + "loss": 0.3071, + "num_tokens": 808898704.0, + "reward": 1.74560546875, + "reward_std": 0.6147333383560181, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.79296875, + "rewards/format_reward/std": 0.40557438135147095, + "rewards/tag_count_reward/mean": 0.89794921875, + "rewards/tag_count_reward/std": 0.22220362722873688, + "step": 1436 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.068359375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2013.0, + "completions/mean_length": 883.8984375, + "completions/mean_terminated_length": 798.482177734375, + "completions/min_length": 190.0, + "completions/min_terminated_length": 190.0, + "epoch": 0.49056925834257914, + "grad_norm": 2.6624701023101807, + "kl": 7.65625, + "learning_rate": 6.428784384005789e-07, + "loss": 0.4915, + "num_tokens": 809430764.0, + "reward": 1.7841796875, + "reward_std": 0.6037580966949463, + "rewards/accuracy_reward/mean": 0.10282257944345474, + "rewards/accuracy_reward/std": 0.30403366684913635, + "rewards/format_reward/mean": 0.79296875, + "rewards/format_reward/std": 0.40557438135147095, + "rewards/tag_count_reward/mean": 0.8916015625, + "rewards/tag_count_reward/std": 0.23191487789154053, + "step": 1437 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2019.0, + "completions/mean_length": 1002.501953125, + "completions/mean_terminated_length": 904.2073364257812, + "completions/min_length": 223.0, + "completions/min_terminated_length": 223.0, + "epoch": 0.49091064265596995, + "grad_norm": 1.8498705625534058, + "kl": 7.9140625, + "learning_rate": 6.423538078963299e-07, + "loss": 0.4687, + "num_tokens": 810020845.0, + "reward": 1.6845703125, + "reward_std": 0.6515914797782898, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.751953125, + "rewards/format_reward/std": 0.4323015511035919, + "rewards/tag_count_reward/mean": 0.8740234375, + "rewards/tag_count_reward/std": 0.2402685582637787, + "step": 1438 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1969.0, + "completions/mean_length": 883.509765625, + "completions/mean_terminated_length": 805.8771362304688, + "completions/min_length": 193.0, + "completions/min_terminated_length": 193.0, + "epoch": 0.49125202696936077, + "grad_norm": 1.4366450309753418, + "kl": 6.3203125, + "learning_rate": 6.418290463127423e-07, + "loss": 0.4229, + "num_tokens": 810556290.0, + "reward": 1.8056640625, + "reward_std": 0.617745041847229, + "rewards/accuracy_reward/mean": 0.115234375, + "rewards/accuracy_reward/std": 0.3196168541908264, + "rewards/format_reward/mean": 0.79296875, + "rewards/format_reward/std": 0.40557438135147095, + "rewards/tag_count_reward/mean": 0.8974609375, + "rewards/tag_count_reward/std": 0.21948480606079102, + "step": 1439 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 876.6015625, + "completions/mean_terminated_length": 821.5050659179688, + "completions/min_length": 148.0, + "completions/min_terminated_length": 148.0, + "epoch": 0.4915934112827516, + "grad_norm": 1.4568078517913818, + "kl": 4.9609375, + "learning_rate": 6.413041543946192e-07, + "loss": 0.3174, + "num_tokens": 811087878.0, + "reward": 1.74755859375, + "reward_std": 0.5549442172050476, + "rewards/accuracy_reward/mean": 0.03427419438958168, + "rewards/accuracy_reward/std": 0.18211629986763, + "rewards/format_reward/mean": 0.806640625, + "rewards/format_reward/std": 0.39531853795051575, + "rewards/tag_count_reward/mean": 0.90771484375, + "rewards/tag_count_reward/std": 0.2048913985490799, + "step": 1440 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 892.8046875, + "completions/mean_terminated_length": 831.0040893554688, + "completions/min_length": 59.0, + "completions/min_terminated_length": 59.0, + "epoch": 0.49193479559614234, + "grad_norm": 2.4246034622192383, + "kl": 6.1875, + "learning_rate": 6.407791328869488e-07, + "loss": 0.3727, + "num_tokens": 811626754.0, + "reward": 1.68408203125, + "reward_std": 0.637832760810852, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.736328125, + "rewards/format_reward/std": 0.4410543739795685, + "rewards/tag_count_reward/mean": 0.88525390625, + "rewards/tag_count_reward/std": 0.21928171813488007, + "step": 1441 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 860.3671875, + "completions/mean_terminated_length": 817.0931396484375, + "completions/min_length": 115.0, + "completions/min_terminated_length": 115.0, + "epoch": 0.49227617990953315, + "grad_norm": 1.6708537340164185, + "kl": 5.96484375, + "learning_rate": 6.402539825349032e-07, + "loss": 0.3627, + "num_tokens": 812145854.0, + "reward": 1.7451171875, + "reward_std": 0.6325790286064148, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.767578125, + "rewards/format_reward/std": 0.42278963327407837, + "rewards/tag_count_reward/mean": 0.8955078125, + "rewards/tag_count_reward/std": 0.21346396207809448, + "step": 1442 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 859.701171875, + "completions/mean_terminated_length": 808.8778686523438, + "completions/min_length": 119.0, + "completions/min_terminated_length": 119.0, + "epoch": 0.49261756422292396, + "grad_norm": 1.6362295150756836, + "kl": 5.0234375, + "learning_rate": 6.397287040838367e-07, + "loss": 0.2981, + "num_tokens": 812658821.0, + "reward": 1.8720703125, + "reward_std": 0.6203747987747192, + "rewards/accuracy_reward/mean": 0.142578125, + "rewards/accuracy_reward/std": 0.3499840497970581, + "rewards/format_reward/mean": 0.822265625, + "rewards/format_reward/std": 0.3826628625392914, + "rewards/tag_count_reward/mean": 0.9072265625, + "rewards/tag_count_reward/std": 0.20074841380119324, + "step": 1443 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1991.0, + "completions/mean_length": 860.16796875, + "completions/mean_terminated_length": 799.1909790039062, + "completions/min_length": 193.0, + "completions/min_terminated_length": 193.0, + "epoch": 0.4929589485363148, + "grad_norm": 2.229505777359009, + "kl": 5.890625, + "learning_rate": 6.392032982792865e-07, + "loss": 0.3685, + "num_tokens": 813171643.0, + "reward": 1.728515625, + "reward_std": 0.5821982622146606, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.77734375, + "rewards/format_reward/std": 0.41643625497817993, + "rewards/tag_count_reward/mean": 0.8984375, + "rewards/tag_count_reward/std": 0.20497123897075653, + "step": 1444 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1993.0, + "completions/mean_length": 884.6328125, + "completions/mean_terminated_length": 807.0750122070312, + "completions/min_length": 69.0, + "completions/min_terminated_length": 69.0, + "epoch": 0.49330033284970554, + "grad_norm": 1.5415078401565552, + "kl": 6.046875, + "learning_rate": 6.386777658669698e-07, + "loss": 0.3889, + "num_tokens": 813709199.0, + "reward": 1.7314453125, + "reward_std": 0.584844172000885, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.77734375, + "rewards/format_reward/std": 0.41643625497817993, + "rewards/tag_count_reward/mean": 0.8974609375, + "rewards/tag_count_reward/std": 0.2166806012392044, + "step": 1445 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 882.5390625, + "completions/mean_terminated_length": 837.6226806640625, + "completions/min_length": 118.0, + "completions/min_terminated_length": 118.0, + "epoch": 0.49364171716309635, + "grad_norm": 1.8591444492340088, + "kl": 4.578125, + "learning_rate": 6.38152107592784e-07, + "loss": 0.3006, + "num_tokens": 814235235.0, + "reward": 1.87451171875, + "reward_std": 0.5980396270751953, + "rewards/accuracy_reward/mean": 0.14516128599643707, + "rewards/accuracy_reward/std": 0.3526190221309662, + "rewards/format_reward/mean": 0.818359375, + "rewards/format_reward/std": 0.38592514395713806, + "rewards/tag_count_reward/mean": 0.91552734375, + "rewards/tag_count_reward/std": 0.19738534092903137, + "step": 1446 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1973.0, + "completions/mean_length": 918.84375, + "completions/mean_terminated_length": 855.9835205078125, + "completions/min_length": 229.0, + "completions/min_terminated_length": 229.0, + "epoch": 0.49398310147648716, + "grad_norm": 1.9435086250305176, + "kl": 5.421875, + "learning_rate": 6.376263242028048e-07, + "loss": 0.3275, + "num_tokens": 814781123.0, + "reward": 1.7568359375, + "reward_std": 0.5942944288253784, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.802734375, + "rewards/format_reward/std": 0.3983237147331238, + "rewards/tag_count_reward/mean": 0.9052734375, + "rewards/tag_count_reward/std": 0.20881156623363495, + "step": 1447 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1965.0, + "completions/mean_length": 814.65234375, + "completions/mean_terminated_length": 753.995849609375, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "epoch": 0.494324485789878, + "grad_norm": 1.8552683591842651, + "kl": 5.2578125, + "learning_rate": 6.371004164432853e-07, + "loss": 0.3284, + "num_tokens": 815281457.0, + "reward": 1.80126953125, + "reward_std": 0.5525183081626892, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.8125, + "rewards/format_reward/std": 0.39069411158561707, + "rewards/tag_count_reward/mean": 0.91455078125, + "rewards/tag_count_reward/std": 0.20247536897659302, + "step": 1448 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2033.0, + "completions/mean_length": 859.658203125, + "completions/mean_terminated_length": 818.8464965820312, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "epoch": 0.49466587010326873, + "grad_norm": 1.342028260231018, + "kl": 5.1875, + "learning_rate": 6.365743850606555e-07, + "loss": 0.3233, + "num_tokens": 815793154.0, + "reward": 1.80224609375, + "reward_std": 0.5543564558029175, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.822265625, + "rewards/format_reward/std": 0.3826628625392914, + "rewards/tag_count_reward/mean": 0.91748046875, + "rewards/tag_count_reward/std": 0.19882753491401672, + "step": 1449 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 841.158203125, + "completions/mean_terminated_length": 786.9734497070312, + "completions/min_length": 113.0, + "completions/min_terminated_length": 113.0, + "epoch": 0.49500725441665955, + "grad_norm": 1.3957794904708862, + "kl": 6.26953125, + "learning_rate": 6.360482308015209e-07, + "loss": 0.3895, + "num_tokens": 816302019.0, + "reward": 1.744140625, + "reward_std": 0.6250890493392944, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.765625, + "rewards/format_reward/std": 0.42402184009552, + "rewards/tag_count_reward/mean": 0.892578125, + "rewards/tag_count_reward/std": 0.21543563902378082, + "step": 1450 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 813.330078125, + "completions/mean_terminated_length": 755.2576293945312, + "completions/min_length": 8.0, + "completions/min_terminated_length": 8.0, + "epoch": 0.49534863873005036, + "grad_norm": 2.0098752975463867, + "kl": 5.5234375, + "learning_rate": 6.35521954412661e-07, + "loss": 0.3354, + "num_tokens": 816805356.0, + "reward": 1.806640625, + "reward_std": 0.5330133438110352, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.828125, + "rewards/format_reward/std": 0.3776407241821289, + "rewards/tag_count_reward/mean": 0.9140625, + "rewards/tag_count_reward/std": 0.1976810246706009, + "step": 1451 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1937.0, + "completions/mean_length": 828.271484375, + "completions/mean_terminated_length": 781.263671875, + "completions/min_length": 129.0, + "completions/min_terminated_length": 129.0, + "epoch": 0.4956900230434412, + "grad_norm": 1.8008296489715576, + "kl": 5.1953125, + "learning_rate": 6.34995556641029e-07, + "loss": 0.3325, + "num_tokens": 817309447.0, + "reward": 1.81640625, + "reward_std": 0.5670454502105713, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.822265625, + "rewards/format_reward/std": 0.3826628625392914, + "rewards/tag_count_reward/mean": 0.916015625, + "rewards/tag_count_reward/std": 0.189053013920784, + "step": 1452 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2004.0, + "completions/mean_length": 813.849609375, + "completions/mean_terminated_length": 761.065185546875, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "epoch": 0.49603140735683193, + "grad_norm": 1.881151795387268, + "kl": 5.02734375, + "learning_rate": 6.344690382337503e-07, + "loss": 0.3325, + "num_tokens": 817805530.0, + "reward": 1.83740234375, + "reward_std": 0.5921496748924255, + "rewards/accuracy_reward/mean": 0.1171875, + "rewards/accuracy_reward/std": 0.32195815443992615, + "rewards/format_reward/mean": 0.810546875, + "rewards/format_reward/std": 0.3922513723373413, + "rewards/tag_count_reward/mean": 0.90966796875, + "rewards/tag_count_reward/std": 0.20155774056911469, + "step": 1453 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1987.0, + "completions/mean_length": 882.619140625, + "completions/mean_terminated_length": 837.7058715820312, + "completions/min_length": 101.0, + "completions/min_terminated_length": 101.0, + "epoch": 0.49637279167022275, + "grad_norm": 1.026684284210205, + "kl": 5.52734375, + "learning_rate": 6.339423999381216e-07, + "loss": 0.3304, + "num_tokens": 818334375.0, + "reward": 1.7353515625, + "reward_std": 0.6159314513206482, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.78515625, + "rewards/format_reward/std": 0.4111155867576599, + "rewards/tag_count_reward/mean": 0.8974609375, + "rewards/tag_count_reward/std": 0.20328371226787567, + "step": 1454 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1980.0, + "completions/mean_length": 783.880859375, + "completions/mean_terminated_length": 750.9478759765625, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "epoch": 0.49671417598361356, + "grad_norm": 2.7136144638061523, + "kl": 4.9375, + "learning_rate": 6.334156425016091e-07, + "loss": 0.3283, + "num_tokens": 818810810.0, + "reward": 1.84033203125, + "reward_std": 0.5457237958908081, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.8359375, + "rewards/format_reward/std": 0.37069445848464966, + "rewards/tag_count_reward/mean": 0.92041015625, + "rewards/tag_count_reward/std": 0.1893402636051178, + "step": 1455 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2023.0, + "completions/mean_length": 839.83984375, + "completions/mean_terminated_length": 795.8178100585938, + "completions/min_length": 190.0, + "completions/min_terminated_length": 190.0, + "epoch": 0.4970555602970044, + "grad_norm": 1.8540130853652954, + "kl": 4.953125, + "learning_rate": 6.328887666718493e-07, + "loss": 0.3185, + "num_tokens": 819316904.0, + "reward": 1.8076171875, + "reward_std": 0.5355762243270874, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.8359375, + "rewards/format_reward/std": 0.37069445848464966, + "rewards/tag_count_reward/mean": 0.9208984375, + "rewards/tag_count_reward/std": 0.18530340492725372, + "step": 1456 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 819.150390625, + "completions/mean_terminated_length": 782.0623168945312, + "completions/min_length": 171.0, + "completions/min_terminated_length": 171.0, + "epoch": 0.49739694461039513, + "grad_norm": 2.3606650829315186, + "kl": 5.5625, + "learning_rate": 6.323617731966456e-07, + "loss": 0.3095, + "num_tokens": 819824597.0, + "reward": 1.7998046875, + "reward_std": 0.542791485786438, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.8203125, + "rewards/format_reward/std": 0.38430243730545044, + "rewards/tag_count_reward/mean": 0.9150390625, + "rewards/tag_count_reward/std": 0.18468356132507324, + "step": 1457 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.056640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1919.0, + "completions/mean_length": 818.34375, + "completions/mean_terminated_length": 744.5134887695312, + "completions/min_length": 148.0, + "completions/min_terminated_length": 148.0, + "epoch": 0.49773832892378594, + "grad_norm": 3.7965686321258545, + "kl": 6.5625, + "learning_rate": 6.318346628239691e-07, + "loss": 0.3538, + "num_tokens": 820333333.0, + "reward": 1.7490234375, + "reward_std": 0.6212472915649414, + "rewards/accuracy_reward/mean": 0.087890625, + "rewards/accuracy_reward/std": 0.2834126651287079, + "rewards/format_reward/mean": 0.771484375, + "rewards/format_reward/std": 0.4202871024608612, + "rewards/tag_count_reward/mean": 0.8896484375, + "rewards/tag_count_reward/std": 0.21903157234191895, + "step": 1458 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2025.0, + "completions/mean_length": 796.962890625, + "completions/mean_terminated_length": 769.4949951171875, + "completions/min_length": 99.0, + "completions/min_terminated_length": 99.0, + "epoch": 0.49807971323717676, + "grad_norm": 1.7308908700942993, + "kl": 5.609375, + "learning_rate": 6.313074363019565e-07, + "loss": 0.334, + "num_tokens": 820825522.0, + "reward": 1.82958984375, + "reward_std": 0.505888044834137, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.93310546875, + "rewards/tag_count_reward/std": 0.17706766724586487, + "step": 1459 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1984.0, + "completions/mean_length": 829.49609375, + "completions/mean_terminated_length": 787.6484985351562, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "epoch": 0.49842109755056757, + "grad_norm": 1.9581514596939087, + "kl": 4.453125, + "learning_rate": 6.307800943789093e-07, + "loss": 0.2748, + "num_tokens": 821322912.0, + "reward": 1.818359375, + "reward_std": 0.529055118560791, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.927734375, + "rewards/tag_count_reward/std": 0.17598573863506317, + "step": 1460 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1868.0, + "completions/mean_length": 759.083984375, + "completions/mean_terminated_length": 728.1500244140625, + "completions/min_length": 159.0, + "completions/min_terminated_length": 159.0, + "epoch": 0.49876248186395833, + "grad_norm": 1.587014079093933, + "kl": 4.15234375, + "learning_rate": 6.302526378032931e-07, + "loss": 0.2444, + "num_tokens": 821785755.0, + "reward": 1.84375, + "reward_std": 0.5097728967666626, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.849609375, + "rewards/format_reward/std": 0.35780346393585205, + "rewards/tag_count_reward/mean": 0.93359375, + "rewards/tag_count_reward/std": 0.16840559244155884, + "step": 1461 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2019.0, + "completions/mean_length": 812.93359375, + "completions/mean_terminated_length": 767.9312133789062, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "epoch": 0.49910386617734914, + "grad_norm": 1.8485121726989746, + "kl": 4.40625, + "learning_rate": 6.29725067323736e-07, + "loss": 0.2859, + "num_tokens": 822288585.0, + "reward": 1.83740234375, + "reward_std": 0.5184470415115356, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.92529296875, + "rewards/tag_count_reward/std": 0.19196152687072754, + "step": 1462 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2030.0, + "completions/mean_length": 799.4375, + "completions/mean_terminated_length": 764.3373413085938, + "completions/min_length": 107.0, + "completions/min_terminated_length": 107.0, + "epoch": 0.49944525049073996, + "grad_norm": 1.4282054901123047, + "kl": 4.373046875, + "learning_rate": 6.291973836890276e-07, + "loss": 0.2487, + "num_tokens": 822775737.0, + "reward": 1.8603515625, + "reward_std": 0.473154217004776, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.9384765625, + "rewards/tag_count_reward/std": 0.1731034219264984, + "step": 1463 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2001.0, + "completions/mean_length": 760.02734375, + "completions/mean_terminated_length": 739.5833740234375, + "completions/min_length": 141.0, + "completions/min_terminated_length": 141.0, + "epoch": 0.49978663480413077, + "grad_norm": 1.2178758382797241, + "kl": 4.06640625, + "learning_rate": 6.286695876481185e-07, + "loss": 0.2455, + "num_tokens": 823243271.0, + "reward": 1.90478515625, + "reward_std": 0.540515661239624, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.93798828125, + "rewards/tag_count_reward/std": 0.16826865077018738, + "step": 1464 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 841.025390625, + "completions/mean_terminated_length": 797.0465698242188, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "epoch": 0.5001280191175216, + "grad_norm": 1.0786937475204468, + "kl": 5.9921875, + "learning_rate": 6.281416799501187e-07, + "loss": 0.3702, + "num_tokens": 823754836.0, + "reward": 1.82177734375, + "reward_std": 0.5706958770751953, + "rewards/accuracy_reward/mean": 0.06854838877916336, + "rewards/accuracy_reward/std": 0.25293973088264465, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.91552734375, + "rewards/tag_count_reward/std": 0.20408765971660614, + "step": 1465 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1978.0, + "completions/mean_length": 797.427734375, + "completions/mean_terminated_length": 759.68408203125, + "completions/min_length": 121.0, + "completions/min_terminated_length": 121.0, + "epoch": 0.5004694034309124, + "grad_norm": 2.623847723007202, + "kl": 5.453125, + "learning_rate": 6.276136613442964e-07, + "loss": 0.339, + "num_tokens": 824239423.0, + "reward": 1.82373046875, + "reward_std": 0.5397425889968872, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.92529296875, + "rewards/tag_count_reward/std": 0.18874886631965637, + "step": 1466 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1756.0, + "completions/mean_length": 718.177734375, + "completions/mean_terminated_length": 683.5330810546875, + "completions/min_length": 126.0, + "completions/min_terminated_length": 126.0, + "epoch": 0.5008107877443031, + "grad_norm": 2.947995662689209, + "kl": 4.3125, + "learning_rate": 6.270855325800775e-07, + "loss": 0.2933, + "num_tokens": 824685594.0, + "reward": 1.8330078125, + "reward_std": 0.41921019554138184, + "rewards/accuracy_reward/mean": 0.026209676638245583, + "rewards/accuracy_reward/std": 0.1599196493625641, + "rewards/format_reward/mean": 0.8671875, + "rewards/format_reward/std": 0.33970388770103455, + "rewards/tag_count_reward/mean": 0.9404296875, + "rewards/tag_count_reward/std": 0.17166221141815186, + "step": 1467 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2015.0, + "completions/mean_length": 834.5390625, + "completions/mean_terminated_length": 787.7728271484375, + "completions/min_length": 157.0, + "completions/min_terminated_length": 157.0, + "epoch": 0.5011521720576939, + "grad_norm": 1.126750111579895, + "kl": 6.8203125, + "learning_rate": 6.265572944070444e-07, + "loss": 0.4545, + "num_tokens": 825185022.0, + "reward": 1.86865234375, + "reward_std": 0.5702815055847168, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.92529296875, + "rewards/tag_count_reward/std": 0.19699282944202423, + "step": 1468 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1895.0, + "completions/mean_length": 734.4140625, + "completions/mean_terminated_length": 700.1923828125, + "completions/min_length": 52.0, + "completions/min_terminated_length": 52.0, + "epoch": 0.5014935563710847, + "grad_norm": 1.8038650751113892, + "kl": 6.36328125, + "learning_rate": 6.260289475749344e-07, + "loss": 0.4107, + "num_tokens": 825629890.0, + "reward": 1.83984375, + "reward_std": 0.4835994243621826, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.943359375, + "rewards/tag_count_reward/std": 0.16318395733833313, + "step": 1469 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1883.0, + "completions/mean_length": 810.154296875, + "completions/mean_terminated_length": 751.9324951171875, + "completions/min_length": 129.0, + "completions/min_terminated_length": 129.0, + "epoch": 0.5018349406844755, + "grad_norm": 5.53438663482666, + "kl": 9.359375, + "learning_rate": 6.255004928336391e-07, + "loss": 0.5345, + "num_tokens": 826114833.0, + "reward": 1.818359375, + "reward_std": 0.6608902812004089, + "rewards/accuracy_reward/mean": 0.12109375, + "rewards/accuracy_reward/std": 0.3265552520751953, + "rewards/format_reward/mean": 0.798828125, + "rewards/format_reward/std": 0.4012683033943176, + "rewards/tag_count_reward/mean": 0.8984375, + "rewards/tag_count_reward/std": 0.22705358266830444, + "step": 1470 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.060546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2018.0, + "completions/mean_length": 801.361328125, + "completions/mean_terminated_length": 721.0166625976562, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "epoch": 0.5021763249978664, + "grad_norm": 8.603954315185547, + "kl": 10.2109375, + "learning_rate": 6.249719309332036e-07, + "loss": 0.5929, + "num_tokens": 826606874.0, + "reward": 1.81201171875, + "reward_std": 0.5934995412826538, + "rewards/accuracy_reward/mean": 0.12298387289047241, + "rewards/accuracy_reward/std": 0.32875028252601624, + "rewards/format_reward/mean": 0.791015625, + "rewards/format_reward/std": 0.40698084235191345, + "rewards/tag_count_reward/mean": 0.90185546875, + "rewards/tag_count_reward/std": 0.21674399077892303, + "step": 1471 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2001.0, + "completions/mean_length": 809.072265625, + "completions/mean_terminated_length": 766.5232543945312, + "completions/min_length": 5.0, + "completions/min_terminated_length": 5.0, + "epoch": 0.5025177093112572, + "grad_norm": 6.623929977416992, + "kl": 9.34375, + "learning_rate": 6.244432626238245e-07, + "loss": 0.4898, + "num_tokens": 827097119.0, + "reward": 1.740234375, + "reward_std": 0.5760315656661987, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.802734375, + "rewards/format_reward/std": 0.3983237147331238, + "rewards/tag_count_reward/mean": 0.8984375, + "rewards/tag_count_reward/std": 0.2291981726884842, + "step": 1472 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1993.0, + "completions/mean_length": 816.001953125, + "completions/mean_terminated_length": 747.41650390625, + "completions/min_length": 189.0, + "completions/min_terminated_length": 189.0, + "epoch": 0.502859093624648, + "grad_norm": 4.908143520355225, + "kl": 8.875, + "learning_rate": 6.239144886558501e-07, + "loss": 0.5392, + "num_tokens": 827588256.0, + "reward": 1.82763671875, + "reward_std": 0.5490024089813232, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.830078125, + "rewards/format_reward/std": 0.3759314715862274, + "rewards/tag_count_reward/mean": 0.92138671875, + "rewards/tag_count_reward/std": 0.19795092940330505, + "step": 1473 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2012.0, + "completions/mean_length": 797.97265625, + "completions/mean_terminated_length": 755.04248046875, + "completions/min_length": 158.0, + "completions/min_terminated_length": 158.0, + "epoch": 0.5032004779380388, + "grad_norm": 3.7246415615081787, + "kl": 8.203125, + "learning_rate": 6.23385609779778e-07, + "loss": 0.4694, + "num_tokens": 828080786.0, + "reward": 1.75048828125, + "reward_std": 0.5996809005737305, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.78515625, + "rewards/format_reward/std": 0.4111155867576599, + "rewards/tag_count_reward/mean": 0.89697265625, + "rewards/tag_count_reward/std": 0.22064577043056488, + "step": 1474 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1981.0, + "completions/mean_length": 742.259765625, + "completions/mean_terminated_length": 697.4161987304688, + "completions/min_length": 9.0, + "completions/min_terminated_length": 9.0, + "epoch": 0.5035418622514295, + "grad_norm": 1.723221778869629, + "kl": 6.6484375, + "learning_rate": 6.228566267462555e-07, + "loss": 0.4491, + "num_tokens": 828548487.0, + "reward": 1.8017578125, + "reward_std": 0.5765774250030518, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.822265625, + "rewards/format_reward/std": 0.3826628625392914, + "rewards/tag_count_reward/mean": 0.9189453125, + "rewards/tag_count_reward/std": 0.1966511756181717, + "step": 1475 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1878.0, + "completions/mean_length": 853.86328125, + "completions/mean_terminated_length": 768.9246826171875, + "completions/min_length": 152.0, + "completions/min_terminated_length": 152.0, + "epoch": 0.5038832465648203, + "grad_norm": 2.8293213844299316, + "kl": 7.3125, + "learning_rate": 6.22327540306077e-07, + "loss": 0.5051, + "num_tokens": 829065681.0, + "reward": 1.732421875, + "reward_std": 0.590774416923523, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.791015625, + "rewards/format_reward/std": 0.40698084235191345, + "rewards/tag_count_reward/mean": 0.89453125, + "rewards/tag_count_reward/std": 0.22742362320423126, + "step": 1476 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2017.0, + "completions/mean_length": 735.369140625, + "completions/mean_terminated_length": 682.0101318359375, + "completions/min_length": 70.0, + "completions/min_terminated_length": 70.0, + "epoch": 0.5042246308782111, + "grad_norm": 3.193406343460083, + "kl": 5.20703125, + "learning_rate": 6.217983512101838e-07, + "loss": 0.3816, + "num_tokens": 829518894.0, + "reward": 1.84765625, + "reward_std": 0.4867114722728729, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.857421875, + "rewards/format_reward/std": 0.3499840497970581, + "rewards/tag_count_reward/mean": 0.931640625, + "rewards/tag_count_reward/std": 0.188242569565773, + "step": 1477 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2003.0, + "completions/mean_length": 764.109375, + "completions/mean_terminated_length": 722.6935424804688, + "completions/min_length": 32.0, + "completions/min_terminated_length": 32.0, + "epoch": 0.5045660151916019, + "grad_norm": 4.938040733337402, + "kl": 4.45703125, + "learning_rate": 6.212690602096631e-07, + "loss": 0.3255, + "num_tokens": 829986182.0, + "reward": 1.87939453125, + "reward_std": 0.48974287509918213, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.93603515625, + "rewards/tag_count_reward/std": 0.1828918159008026, + "step": 1478 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1788.0, + "completions/mean_length": 800.89453125, + "completions/mean_terminated_length": 744.9020385742188, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "epoch": 0.5049073995049927, + "grad_norm": 1.9067102670669556, + "kl": 6.2265625, + "learning_rate": 6.207396680557468e-07, + "loss": 0.397, + "num_tokens": 830474224.0, + "reward": 1.791015625, + "reward_std": 0.549534797668457, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.83203125, + "rewards/format_reward/std": 0.374204158782959, + "rewards/tag_count_reward/mean": 0.91796875, + "rewards/tag_count_reward/std": 0.20774950087070465, + "step": 1479 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1949.0, + "completions/mean_length": 824.9296875, + "completions/mean_terminated_length": 767.40283203125, + "completions/min_length": 71.0, + "completions/min_terminated_length": 71.0, + "epoch": 0.5052487838183836, + "grad_norm": 1.4719449281692505, + "kl": 7.59375, + "learning_rate": 6.202101754998101e-07, + "loss": 0.4883, + "num_tokens": 830974844.0, + "reward": 1.79345703125, + "reward_std": 0.5807920694351196, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.818359375, + "rewards/format_reward/std": 0.38592514395713806, + "rewards/tag_count_reward/mean": 0.90478515625, + "rewards/tag_count_reward/std": 0.2146575003862381, + "step": 1480 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2023.0, + "completions/mean_length": 859.779296875, + "completions/mean_terminated_length": 785.8236694335938, + "completions/min_length": 70.0, + "completions/min_terminated_length": 70.0, + "epoch": 0.5055901681317744, + "grad_norm": 1.5499788522720337, + "kl": 9.5, + "learning_rate": 6.196805832933709e-07, + "loss": 0.6123, + "num_tokens": 831489627.0, + "reward": 1.7060546875, + "reward_std": 0.6194310784339905, + "rewards/accuracy_reward/mean": 0.02734375, + "rewards/accuracy_reward/std": 0.16324250400066376, + "rewards/format_reward/mean": 0.783203125, + "rewards/format_reward/std": 0.4124660789966583, + "rewards/tag_count_reward/mean": 0.8955078125, + "rewards/tag_count_reward/std": 0.2257176786661148, + "step": 1481 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1846.0, + "completions/mean_length": 763.044921875, + "completions/mean_terminated_length": 729.5691528320312, + "completions/min_length": 72.0, + "completions/min_terminated_length": 72.0, + "epoch": 0.5059315524451652, + "grad_norm": 2.0553057193756104, + "kl": 5.703125, + "learning_rate": 6.191508921880886e-07, + "loss": 0.3149, + "num_tokens": 831953186.0, + "reward": 1.857421875, + "reward_std": 0.5050795674324036, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.857421875, + "rewards/format_reward/std": 0.3499840497970581, + "rewards/tag_count_reward/mean": 0.92578125, + "rewards/tag_count_reward/std": 0.18991030752658844, + "step": 1482 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1725.0, + "completions/mean_length": 859.18359375, + "completions/mean_terminated_length": 798.1560668945312, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "epoch": 0.5062729367585559, + "grad_norm": 1.0283100605010986, + "kl": 7.671875, + "learning_rate": 6.186211029357625e-07, + "loss": 0.4849, + "num_tokens": 832475232.0, + "reward": 1.84228515625, + "reward_std": 0.6034194231033325, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.82421875, + "rewards/format_reward/std": 0.3810062110424042, + "rewards/tag_count_reward/mean": 0.91259765625, + "rewards/tag_count_reward/std": 0.20937539637088776, + "step": 1483 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1976.0, + "completions/mean_length": 799.09375, + "completions/mean_terminated_length": 750.96142578125, + "completions/min_length": 151.0, + "completions/min_terminated_length": 151.0, + "epoch": 0.5066143210719467, + "grad_norm": 2.066352605819702, + "kl": 6.2890625, + "learning_rate": 6.180912162883318e-07, + "loss": 0.4158, + "num_tokens": 832958992.0, + "reward": 1.8359375, + "reward_std": 0.45680689811706543, + "rewards/accuracy_reward/mean": 0.024193547666072845, + "rewards/accuracy_reward/std": 0.15380479395389557, + "rewards/format_reward/mean": 0.87890625, + "rewards/format_reward/std": 0.3265552520751953, + "rewards/tag_count_reward/mean": 0.93359375, + "rewards/tag_count_reward/std": 0.1895880103111267, + "step": 1484 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2013.0, + "completions/mean_length": 780.0859375, + "completions/mean_terminated_length": 741.8189086914062, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "epoch": 0.5069557053853375, + "grad_norm": 2.2035834789276123, + "kl": 7.03125, + "learning_rate": 6.175612329978737e-07, + "loss": 0.431, + "num_tokens": 833437820.0, + "reward": 1.83984375, + "reward_std": 0.5861037373542786, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.83203125, + "rewards/format_reward/std": 0.374204158782959, + "rewards/tag_count_reward/mean": 0.92578125, + "rewards/tag_count_reward/std": 0.1937359869480133, + "step": 1485 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1951.0, + "completions/mean_length": 825.13671875, + "completions/mean_terminated_length": 775.4268188476562, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "epoch": 0.5072970896987283, + "grad_norm": 1.061333179473877, + "kl": 6.2578125, + "learning_rate": 6.170311538166026e-07, + "loss": 0.3749, + "num_tokens": 833948498.0, + "reward": 1.78759765625, + "reward_std": 0.5507444739341736, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.818359375, + "rewards/format_reward/std": 0.38592514395713806, + "rewards/tag_count_reward/mean": 0.91259765625, + "rewards/tag_count_reward/std": 0.2099587321281433, + "step": 1486 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1940.0, + "completions/mean_length": 851.1640625, + "completions/mean_terminated_length": 794.8711547851562, + "completions/min_length": 178.0, + "completions/min_terminated_length": 178.0, + "epoch": 0.5076384740121191, + "grad_norm": 2.173102617263794, + "kl": 8.109375, + "learning_rate": 6.165009794968687e-07, + "loss": 0.4881, + "num_tokens": 834458966.0, + "reward": 1.7919921875, + "reward_std": 0.5524708032608032, + "rewards/accuracy_reward/mean": 0.04032257944345474, + "rewards/accuracy_reward/std": 0.19691328704357147, + "rewards/format_reward/mean": 0.833984375, + "rewards/format_reward/std": 0.3724585771560669, + "rewards/tag_count_reward/mean": 0.9189453125, + "rewards/tag_count_reward/std": 0.20217134058475494, + "step": 1487 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1973.0, + "completions/mean_length": 863.71484375, + "completions/mean_terminated_length": 802.919921875, + "completions/min_length": 153.0, + "completions/min_terminated_length": 153.0, + "epoch": 0.50797985832551, + "grad_norm": 3.464843273162842, + "kl": 9.125, + "learning_rate": 6.159707107911575e-07, + "loss": 0.5288, + "num_tokens": 834979844.0, + "reward": 1.80908203125, + "reward_std": 0.627461314201355, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.814453125, + "rewards/format_reward/std": 0.38912075757980347, + "rewards/tag_count_reward/mean": 0.91259765625, + "rewards/tag_count_reward/std": 0.21342535316944122, + "step": 1488 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2003.0, + "completions/mean_length": 811.58984375, + "completions/mean_terminated_length": 753.435546875, + "completions/min_length": 236.0, + "completions/min_terminated_length": 236.0, + "epoch": 0.5083212426389008, + "grad_norm": 2.9201884269714355, + "kl": 9.1328125, + "learning_rate": 6.154403484520887e-07, + "loss": 0.5415, + "num_tokens": 835473794.0, + "reward": 1.810546875, + "reward_std": 0.5905405282974243, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.828125, + "rewards/format_reward/std": 0.3776407241821289, + "rewards/tag_count_reward/mean": 0.90625, + "rewards/tag_count_reward/std": 0.21728172898292542, + "step": 1489 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 810.4453125, + "completions/mean_terminated_length": 762.75048828125, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "epoch": 0.5086626269522916, + "grad_norm": 1.2315624952316284, + "kl": 6.859375, + "learning_rate": 6.149098932324145e-07, + "loss": 0.3881, + "num_tokens": 835964902.0, + "reward": 1.8662109375, + "reward_std": 0.5907376408576965, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.9208984375, + "rewards/tag_count_reward/std": 0.19744645059108734, + "step": 1490 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.056640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1966.0, + "completions/mean_length": 848.46484375, + "completions/mean_terminated_length": 776.4430541992188, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "epoch": 0.5090040112656823, + "grad_norm": 1.9907715320587158, + "kl": 8.15625, + "learning_rate": 6.143793458850188e-07, + "loss": 0.5139, + "num_tokens": 836471988.0, + "reward": 1.80322265625, + "reward_std": 0.5664142370223999, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.8203125, + "rewards/format_reward/std": 0.38430243730545044, + "rewards/tag_count_reward/mean": 0.90869140625, + "rewards/tag_count_reward/std": 0.21521764993667603, + "step": 1491 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2021.0, + "completions/mean_length": 934.6953125, + "completions/mean_terminated_length": 870.2892456054688, + "completions/min_length": 102.0, + "completions/min_terminated_length": 102.0, + "epoch": 0.5093453955790731, + "grad_norm": 0.9177165031433105, + "kl": 7.0234375, + "learning_rate": 6.13848707162917e-07, + "loss": 0.4453, + "num_tokens": 837022008.0, + "reward": 1.77197265625, + "reward_std": 0.5535103678703308, + "rewards/accuracy_reward/mean": 0.058467742055654526, + "rewards/accuracy_reward/std": 0.23486268520355225, + "rewards/format_reward/mean": 0.80859375, + "rewards/format_reward/std": 0.3937928080558777, + "rewards/tag_count_reward/mean": 0.90673828125, + "rewards/tag_count_reward/std": 0.20741750299930573, + "step": 1492 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1976.0, + "completions/mean_length": 886.91796875, + "completions/mean_terminated_length": 824.8024291992188, + "completions/min_length": 84.0, + "completions/min_terminated_length": 84.0, + "epoch": 0.5096867798924639, + "grad_norm": 1.5554848909378052, + "kl": 8.2734375, + "learning_rate": 6.133179778192533e-07, + "loss": 0.4791, + "num_tokens": 837550670.0, + "reward": 1.74462890625, + "reward_std": 0.6169675588607788, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.77734375, + "rewards/format_reward/std": 0.41643625497817993, + "rewards/tag_count_reward/mean": 0.89501953125, + "rewards/tag_count_reward/std": 0.22358404099941254, + "step": 1493 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 883.537109375, + "completions/mean_terminated_length": 826.2683715820312, + "completions/min_length": 129.0, + "completions/min_terminated_length": 129.0, + "epoch": 0.5100281642058547, + "grad_norm": 1.5476243495941162, + "kl": 6.25390625, + "learning_rate": 6.127871586073012e-07, + "loss": 0.3769, + "num_tokens": 838069713.0, + "reward": 1.84033203125, + "reward_std": 0.5659228563308716, + "rewards/accuracy_reward/mean": 0.1015625, + "rewards/accuracy_reward/std": 0.30236753821372986, + "rewards/format_reward/mean": 0.82421875, + "rewards/format_reward/std": 0.3810062110424042, + "rewards/tag_count_reward/mean": 0.91455078125, + "rewards/tag_count_reward/std": 0.20606790482997894, + "step": 1494 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1992.0, + "completions/mean_length": 818.41015625, + "completions/mean_terminated_length": 773.6072998046875, + "completions/min_length": 41.0, + "completions/min_terminated_length": 41.0, + "epoch": 0.5103695485192455, + "grad_norm": 3.4402146339416504, + "kl": 5.52734375, + "learning_rate": 6.122562502804614e-07, + "loss": 0.3724, + "num_tokens": 838570083.0, + "reward": 1.84912109375, + "reward_std": 0.5798235535621643, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.91943359375, + "rewards/tag_count_reward/std": 0.19901487231254578, + "step": 1495 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1980.0, + "completions/mean_length": 825.552734375, + "completions/mean_terminated_length": 768.05517578125, + "completions/min_length": 151.0, + "completions/min_terminated_length": 151.0, + "epoch": 0.5107109328326364, + "grad_norm": 1.9902147054672241, + "kl": 5.7890625, + "learning_rate": 6.117252535922611e-07, + "loss": 0.3707, + "num_tokens": 839060926.0, + "reward": 1.7998046875, + "reward_std": 0.5655973553657532, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.8125, + "rewards/format_reward/std": 0.39069411158561707, + "rewards/tag_count_reward/mean": 0.9091796875, + "rewards/tag_count_reward/std": 0.2128543108701706, + "step": 1496 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 831.1640625, + "completions/mean_terminated_length": 781.6991577148438, + "completions/min_length": 130.0, + "completions/min_terminated_length": 130.0, + "epoch": 0.5110523171460272, + "grad_norm": 1.3155391216278076, + "kl": 6.11328125, + "learning_rate": 6.111941692963531e-07, + "loss": 0.3642, + "num_tokens": 839563186.0, + "reward": 1.84375, + "reward_std": 0.5692417621612549, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.921875, + "rewards/tag_count_reward/std": 0.19282633066177368, + "step": 1497 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.056640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 829.890625, + "completions/mean_terminated_length": 756.753662109375, + "completions/min_length": 178.0, + "completions/min_terminated_length": 178.0, + "epoch": 0.511393701459418, + "grad_norm": 0.9072387218475342, + "kl": 6.34375, + "learning_rate": 6.106629981465142e-07, + "loss": 0.3769, + "num_tokens": 840072794.0, + "reward": 1.7841796875, + "reward_std": 0.6087607145309448, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.791015625, + "rewards/format_reward/std": 0.40698084235191345, + "rewards/tag_count_reward/mean": 0.8994140625, + "rewards/tag_count_reward/std": 0.21871723234653473, + "step": 1498 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 795.34375, + "completions/mean_terminated_length": 767.84033203125, + "completions/min_length": 70.0, + "completions/min_terminated_length": 70.0, + "epoch": 0.5117350857728087, + "grad_norm": 1.411439061164856, + "kl": 6.125, + "learning_rate": 6.101317408966451e-07, + "loss": 0.3668, + "num_tokens": 840557850.0, + "reward": 1.81591796875, + "reward_std": 0.5884963274002075, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.818359375, + "rewards/format_reward/std": 0.38592514395713806, + "rewards/tag_count_reward/mean": 0.91748046875, + "rewards/tag_count_reward/std": 0.1925777792930603, + "step": 1499 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2003.0, + "completions/mean_length": 870.6015625, + "completions/mean_terminated_length": 802.487548828125, + "completions/min_length": 89.0, + "completions/min_terminated_length": 89.0, + "epoch": 0.5120764700861995, + "grad_norm": 2.008626937866211, + "kl": 8.2109375, + "learning_rate": 6.096003983007679e-07, + "loss": 0.4929, + "num_tokens": 841079662.0, + "reward": 1.78076171875, + "reward_std": 0.5997934341430664, + "rewards/accuracy_reward/mean": 0.087890625, + "rewards/accuracy_reward/std": 0.2834126651287079, + "rewards/format_reward/mean": 0.802734375, + "rewards/format_reward/std": 0.3983237147331238, + "rewards/tag_count_reward/mean": 0.89013671875, + "rewards/tag_count_reward/std": 0.2288304716348648, + "step": 1500 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2022.0, + "completions/mean_length": 754.94140625, + "completions/mean_terminated_length": 710.5333862304688, + "completions/min_length": 114.0, + "completions/min_terminated_length": 114.0, + "epoch": 0.5124178543995903, + "grad_norm": 1.2399321794509888, + "kl": 6.19921875, + "learning_rate": 6.090689711130263e-07, + "loss": 0.3782, + "num_tokens": 841540032.0, + "reward": 1.845703125, + "reward_std": 0.5702787637710571, + "rewards/accuracy_reward/mean": 0.109375, + "rewards/accuracy_reward/std": 0.31241437792778015, + "rewards/format_reward/mean": 0.82421875, + "rewards/format_reward/std": 0.3810062110424042, + "rewards/tag_count_reward/mean": 0.912109375, + "rewards/tag_count_reward/std": 0.20473802089691162, + "step": 1501 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1961.0, + "completions/mean_length": 814.29296875, + "completions/mean_terminated_length": 756.2658081054688, + "completions/min_length": 113.0, + "completions/min_terminated_length": 113.0, + "epoch": 0.5127592387129811, + "grad_norm": 1.3162981271743774, + "kl": 6.515625, + "learning_rate": 6.085374600876842e-07, + "loss": 0.4093, + "num_tokens": 842031254.0, + "reward": 1.78759765625, + "reward_std": 0.5792131423950195, + "rewards/accuracy_reward/mean": 0.07661290466785431, + "rewards/accuracy_reward/std": 0.2662447690963745, + "rewards/format_reward/mean": 0.802734375, + "rewards/format_reward/std": 0.3983237147331238, + "rewards/tag_count_reward/mean": 0.91064453125, + "rewards/tag_count_reward/std": 0.20737142860889435, + "step": 1502 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 862.115234375, + "completions/mean_terminated_length": 823.86083984375, + "completions/min_length": 36.0, + "completions/min_terminated_length": 36.0, + "epoch": 0.5131006230263719, + "grad_norm": 1.4820775985717773, + "kl": 5.3828125, + "learning_rate": 6.080058659791241e-07, + "loss": 0.335, + "num_tokens": 842546865.0, + "reward": 1.79736328125, + "reward_std": 0.5385974645614624, + "rewards/accuracy_reward/mean": 0.037109375, + "rewards/accuracy_reward/std": 0.18921469151973724, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.92041015625, + "rewards/tag_count_reward/std": 0.19190676510334015, + "step": 1503 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2002.0, + "completions/mean_length": 823.140625, + "completions/mean_terminated_length": 781.0747680664062, + "completions/min_length": 21.0, + "completions/min_terminated_length": 21.0, + "epoch": 0.5134420073397628, + "grad_norm": 1.1283048391342163, + "kl": 6.42578125, + "learning_rate": 6.074741895418466e-07, + "loss": 0.4094, + "num_tokens": 843043833.0, + "reward": 1.78857421875, + "reward_std": 0.5742803812026978, + "rewards/accuracy_reward/mean": 0.05645161122083664, + "rewards/accuracy_reward/std": 0.23102474212646484, + "rewards/format_reward/mean": 0.826171875, + "rewards/format_reward/std": 0.3793322443962097, + "rewards/tag_count_reward/mean": 0.90771484375, + "rewards/tag_count_reward/std": 0.21193371713161469, + "step": 1504 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1944.0, + "completions/mean_length": 830.80078125, + "completions/mean_terminated_length": 781.3211059570312, + "completions/min_length": 78.0, + "completions/min_terminated_length": 78.0, + "epoch": 0.5137833916531536, + "grad_norm": 2.193281650543213, + "kl": 5.4375, + "learning_rate": 6.069424315304693e-07, + "loss": 0.3357, + "num_tokens": 843549619.0, + "reward": 1.83837890625, + "reward_std": 0.5884426236152649, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.82421875, + "rewards/format_reward/std": 0.3810062110424042, + "rewards/tag_count_reward/mean": 0.91845703125, + "rewards/tag_count_reward/std": 0.19488608837127686, + "step": 1505 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1987.0, + "completions/mean_length": 870.666015625, + "completions/mean_terminated_length": 797.3880004882812, + "completions/min_length": 199.0, + "completions/min_terminated_length": 199.0, + "epoch": 0.5141247759665444, + "grad_norm": 2.0715394020080566, + "kl": 6.109375, + "learning_rate": 6.064105926997251e-07, + "loss": 0.4299, + "num_tokens": 844070584.0, + "reward": 1.818359375, + "reward_std": 0.5174299478530884, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.9140625, + "rewards/tag_count_reward/std": 0.2096906155347824, + "step": 1506 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1787.0, + "completions/mean_length": 823.134765625, + "completions/mean_terminated_length": 760.2567138671875, + "completions/min_length": 152.0, + "completions/min_terminated_length": 152.0, + "epoch": 0.5144661602799352, + "grad_norm": 2.2035629749298096, + "kl": 6.359375, + "learning_rate": 6.058786738044626e-07, + "loss": 0.4295, + "num_tokens": 844560749.0, + "reward": 1.88525390625, + "reward_std": 0.5651792883872986, + "rewards/accuracy_reward/mean": 0.115234375, + "rewards/accuracy_reward/std": 0.3196168541908264, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.92431640625, + "rewards/tag_count_reward/std": 0.20031657814979553, + "step": 1507 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1922.0, + "completions/mean_length": 833.3359375, + "completions/mean_terminated_length": 809.1394653320312, + "completions/min_length": 141.0, + "completions/min_terminated_length": 141.0, + "epoch": 0.5148075445933259, + "grad_norm": 1.364289402961731, + "kl": 5.0703125, + "learning_rate": 6.053466755996427e-07, + "loss": 0.3246, + "num_tokens": 845066505.0, + "reward": 1.86083984375, + "reward_std": 0.5406478643417358, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.849609375, + "rewards/format_reward/std": 0.35780346393585205, + "rewards/tag_count_reward/mean": 0.93310546875, + "rewards/tag_count_reward/std": 0.17912793159484863, + "step": 1508 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2015.0, + "completions/mean_length": 813.23828125, + "completions/mean_terminated_length": 760.427734375, + "completions/min_length": 11.0, + "completions/min_terminated_length": 11.0, + "epoch": 0.5151489289067167, + "grad_norm": 4.429231643676758, + "kl": 9.5078125, + "learning_rate": 6.0481459884034e-07, + "loss": 0.5563, + "num_tokens": 845555907.0, + "reward": 1.74072265625, + "reward_std": 0.5743678212165833, + "rewards/accuracy_reward/mean": 0.029296875, + "rewards/accuracy_reward/std": 0.16880230605602264, + "rewards/format_reward/mean": 0.814453125, + "rewards/format_reward/std": 0.38912075757980347, + "rewards/tag_count_reward/mean": 0.89697265625, + "rewards/tag_count_reward/std": 0.2282744199037552, + "step": 1509 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.072265625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1984.0, + "completions/mean_length": 877.82421875, + "completions/mean_terminated_length": 786.6736450195312, + "completions/min_length": 185.0, + "completions/min_terminated_length": 185.0, + "epoch": 0.5154903132201075, + "grad_norm": 4.857111930847168, + "kl": 11.640625, + "learning_rate": 6.042824442817399e-07, + "loss": 0.7312, + "num_tokens": 846076745.0, + "reward": 1.75830078125, + "reward_std": 0.6979130506515503, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.78515625, + "rewards/format_reward/std": 0.4111155867576599, + "rewards/tag_count_reward/mean": 0.88916015625, + "rewards/tag_count_reward/std": 0.2336527705192566, + "step": 1510 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2025.0, + "completions/mean_length": 793.458984375, + "completions/mean_terminated_length": 745.1094970703125, + "completions/min_length": 215.0, + "completions/min_terminated_length": 215.0, + "epoch": 0.5158316975334983, + "grad_norm": 5.492738723754883, + "kl": 10.765625, + "learning_rate": 6.037502126791386e-07, + "loss": 0.6317, + "num_tokens": 846556900.0, + "reward": 1.72802734375, + "reward_std": 0.6071256995201111, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.783203125, + "rewards/format_reward/std": 0.4124660789966583, + "rewards/tag_count_reward/mean": 0.89404296875, + "rewards/tag_count_reward/std": 0.22421565651893616, + "step": 1511 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2041.0, + "completions/mean_length": 838.787109375, + "completions/mean_terminated_length": 774.0966796875, + "completions/min_length": 223.0, + "completions/min_terminated_length": 223.0, + "epoch": 0.5161730818468891, + "grad_norm": 6.562992095947266, + "kl": 9.859375, + "learning_rate": 6.032179047879413e-07, + "loss": 0.5575, + "num_tokens": 847060663.0, + "reward": 1.7548828125, + "reward_std": 0.5794814229011536, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.796875, + "rewards/format_reward/std": 0.4027182459831238, + "rewards/tag_count_reward/mean": 0.8896484375, + "rewards/tag_count_reward/std": 0.23257318139076233, + "step": 1512 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 814.951171875, + "completions/mean_terminated_length": 754.3093872070312, + "completions/min_length": 144.0, + "completions/min_terminated_length": 144.0, + "epoch": 0.51651446616028, + "grad_norm": 3.621229410171509, + "kl": 9.2265625, + "learning_rate": 6.026855213636619e-07, + "loss": 0.5589, + "num_tokens": 847558846.0, + "reward": 1.7900390625, + "reward_std": 0.6148122549057007, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.802734375, + "rewards/format_reward/std": 0.3983237147331238, + "rewards/tag_count_reward/mean": 0.9033203125, + "rewards/tag_count_reward/std": 0.21824489533901215, + "step": 1513 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 818.384765625, + "completions/mean_terminated_length": 752.6028442382812, + "completions/min_length": 154.0, + "completions/min_terminated_length": 154.0, + "epoch": 0.5168558504736708, + "grad_norm": 1.4343737363815308, + "kl": 8.203125, + "learning_rate": 6.021530631619213e-07, + "loss": 0.5196, + "num_tokens": 848049523.0, + "reward": 1.75732421875, + "reward_std": 0.6363678574562073, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.806640625, + "rewards/format_reward/std": 0.39531853795051575, + "rewards/tag_count_reward/mean": 0.89208984375, + "rewards/tag_count_reward/std": 0.23554269969463348, + "step": 1514 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1998.0, + "completions/mean_length": 852.2578125, + "completions/mean_terminated_length": 783.0826416015625, + "completions/min_length": 171.0, + "completions/min_terminated_length": 171.0, + "epoch": 0.5171972347870616, + "grad_norm": 1.6349339485168457, + "kl": 7.8125, + "learning_rate": 6.016205309384466e-07, + "loss": 0.5089, + "num_tokens": 848560471.0, + "reward": 1.7529296875, + "reward_std": 0.6010799407958984, + "rewards/accuracy_reward/mean": 0.0463709682226181, + "rewards/accuracy_reward/std": 0.21049949526786804, + "rewards/format_reward/mean": 0.806640625, + "rewards/format_reward/std": 0.39531853795051575, + "rewards/tag_count_reward/mean": 0.9013671875, + "rewards/tag_count_reward/std": 0.2179294228553772, + "step": 1515 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.064453125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2030.0, + "completions/mean_length": 849.228515625, + "completions/mean_terminated_length": 766.6409301757812, + "completions/min_length": 143.0, + "completions/min_terminated_length": 143.0, + "epoch": 0.5175386191004523, + "grad_norm": 2.201406240463257, + "kl": 8.15625, + "learning_rate": 6.010879254490695e-07, + "loss": 0.5508, + "num_tokens": 849075580.0, + "reward": 1.72900390625, + "reward_std": 0.6683923006057739, + "rewards/accuracy_reward/mean": 0.07661290466785431, + "rewards/accuracy_reward/std": 0.2662447690963745, + "rewards/format_reward/mean": 0.76953125, + "rewards/format_reward/std": 0.42154473066329956, + "rewards/tag_count_reward/mean": 0.88525390625, + "rewards/tag_count_reward/std": 0.23903003334999084, + "step": 1516 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2012.0, + "completions/mean_length": 786.662109375, + "completions/mean_terminated_length": 730.0305786132812, + "completions/min_length": 144.0, + "completions/min_terminated_length": 144.0, + "epoch": 0.5178800034138431, + "grad_norm": 0.942610502243042, + "kl": 6.5546875, + "learning_rate": 6.005552474497264e-07, + "loss": 0.3864, + "num_tokens": 849555007.0, + "reward": 1.72607421875, + "reward_std": 0.5106169581413269, + "rewards/accuracy_reward/mean": 0.02734375, + "rewards/accuracy_reward/std": 0.16324250400066376, + "rewards/format_reward/mean": 0.791015625, + "rewards/format_reward/std": 0.40698084235191345, + "rewards/tag_count_reward/mean": 0.90771484375, + "rewards/tag_count_reward/std": 0.20726542174816132, + "step": 1517 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 759.005859375, + "completions/mean_terminated_length": 720.1026000976562, + "completions/min_length": 52.0, + "completions/min_terminated_length": 52.0, + "epoch": 0.5182213877272339, + "grad_norm": 2.7662010192871094, + "kl": 5.0625, + "learning_rate": 6.000224976964563e-07, + "loss": 0.3215, + "num_tokens": 850021538.0, + "reward": 1.810546875, + "reward_std": 0.5042428970336914, + "rewards/accuracy_reward/mean": 0.038306452333927155, + "rewards/accuracy_reward/std": 0.19212885200977325, + "rewards/format_reward/mean": 0.849609375, + "rewards/format_reward/std": 0.35780346393585205, + "rewards/tag_count_reward/mean": 0.923828125, + "rewards/tag_count_reward/std": 0.19859731197357178, + "step": 1518 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2029.0, + "completions/mean_length": 823.802734375, + "completions/mean_terminated_length": 781.7596435546875, + "completions/min_length": 63.0, + "completions/min_terminated_length": 63.0, + "epoch": 0.5185627720406247, + "grad_norm": 2.674511671066284, + "kl": 4.62890625, + "learning_rate": 5.994896769453999e-07, + "loss": 0.3008, + "num_tokens": 850525933.0, + "reward": 1.857421875, + "reward_std": 0.513138473033905, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.9296875, + "rewards/tag_count_reward/std": 0.1939331740140915, + "step": 1519 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 861.314453125, + "completions/mean_terminated_length": 810.5601196289062, + "completions/min_length": 192.0, + "completions/min_terminated_length": 192.0, + "epoch": 0.5189041563540155, + "grad_norm": 3.3661677837371826, + "kl": 4.93359375, + "learning_rate": 5.989567859527988e-07, + "loss": 0.3711, + "num_tokens": 851044750.0, + "reward": 1.86083984375, + "reward_std": 0.486860990524292, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.93115234375, + "rewards/tag_count_reward/std": 0.18643119931221008, + "step": 1520 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1985.0, + "completions/mean_length": 837.64453125, + "completions/mean_terminated_length": 762.3112182617188, + "completions/min_length": 196.0, + "completions/min_terminated_length": 196.0, + "epoch": 0.5192455406674064, + "grad_norm": 4.249598979949951, + "kl": 6.265625, + "learning_rate": 5.984238254749946e-07, + "loss": 0.4595, + "num_tokens": 851550600.0, + "reward": 1.78955078125, + "reward_std": 0.5405020117759705, + "rewards/accuracy_reward/mean": 0.033203125, + "rewards/accuracy_reward/std": 0.17934183776378632, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.91845703125, + "rewards/tag_count_reward/std": 0.20764772593975067, + "step": 1521 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.060546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 814.98828125, + "completions/mean_terminated_length": 735.5218505859375, + "completions/min_length": 144.0, + "completions/min_terminated_length": 144.0, + "epoch": 0.5195869249807972, + "grad_norm": 1.2901614904403687, + "kl": 8.71875, + "learning_rate": 5.978907962684267e-07, + "loss": 0.5619, + "num_tokens": 852039490.0, + "reward": 1.7841796875, + "reward_std": 0.5843145847320557, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.810546875, + "rewards/format_reward/std": 0.3922513723373413, + "rewards/tag_count_reward/mean": 0.9052734375, + "rewards/tag_count_reward/std": 0.21910135447978973, + "step": 1522 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 783.240234375, + "completions/mean_terminated_length": 715.5781860351562, + "completions/min_length": 195.0, + "completions/min_terminated_length": 195.0, + "epoch": 0.519928309294188, + "grad_norm": 3.2713301181793213, + "kl": 8.2265625, + "learning_rate": 5.973576990896331e-07, + "loss": 0.4684, + "num_tokens": 852519293.0, + "reward": 1.78466796875, + "reward_std": 0.595682680606842, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.80859375, + "rewards/format_reward/std": 0.3937928080558777, + "rewards/tag_count_reward/mean": 0.89599609375, + "rewards/tag_count_reward/std": 0.23102888464927673, + "step": 1523 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1957.0, + "completions/mean_length": 833.33984375, + "completions/mean_terminated_length": 781.3890380859375, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "epoch": 0.5202696936075787, + "grad_norm": 2.170908212661743, + "kl": 8.73828125, + "learning_rate": 5.968245346952473e-07, + "loss": 0.5018, + "num_tokens": 853015963.0, + "reward": 1.72607421875, + "reward_std": 0.6007155179977417, + "rewards/accuracy_reward/mean": 0.03125, + "rewards/accuracy_reward/std": 0.17416280508041382, + "rewards/format_reward/mean": 0.796875, + "rewards/format_reward/std": 0.4027182459831238, + "rewards/tag_count_reward/mean": 0.89794921875, + "rewards/tag_count_reward/std": 0.2308426946401596, + "step": 1524 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.076171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1840.0, + "completions/mean_length": 868.455078125, + "completions/mean_terminated_length": 771.19873046875, + "completions/min_length": 54.0, + "completions/min_terminated_length": 54.0, + "epoch": 0.5206110779209695, + "grad_norm": 3.0966808795928955, + "kl": 10.578125, + "learning_rate": 5.962913038419988e-07, + "loss": 0.6346, + "num_tokens": 853539348.0, + "reward": 1.7158203125, + "reward_std": 0.6263778805732727, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.794921875, + "rewards/format_reward/std": 0.4041535556316376, + "rewards/tag_count_reward/mean": 0.8818359375, + "rewards/tag_count_reward/std": 0.24867978692054749, + "step": 1525 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 879.61328125, + "completions/mean_terminated_length": 824.658447265625, + "completions/min_length": 5.0, + "completions/min_terminated_length": 5.0, + "epoch": 0.5209524622343603, + "grad_norm": 1.897019624710083, + "kl": 8.2890625, + "learning_rate": 5.957580072867113e-07, + "loss": 0.4556, + "num_tokens": 854069854.0, + "reward": 1.720703125, + "reward_std": 0.6369937658309937, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.78125, + "rewards/format_reward/std": 0.41380295157432556, + "rewards/tag_count_reward/mean": 0.888671875, + "rewards/tag_count_reward/std": 0.2305201143026352, + "step": 1526 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2002.0, + "completions/mean_length": 876.908203125, + "completions/mean_terminated_length": 793.6087646484375, + "completions/min_length": 198.0, + "completions/min_terminated_length": 198.0, + "epoch": 0.5212938465477511, + "grad_norm": 1.3214539289474487, + "kl": 8.0859375, + "learning_rate": 5.952246457863019e-07, + "loss": 0.4706, + "num_tokens": 854593455.0, + "reward": 1.72509765625, + "reward_std": 0.5969668626785278, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.787109375, + "rewards/format_reward/std": 0.409751296043396, + "rewards/tag_count_reward/mean": 0.88916015625, + "rewards/tag_count_reward/std": 0.23207706212997437, + "step": 1527 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.064453125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 864.830078125, + "completions/mean_terminated_length": 783.3173217773438, + "completions/min_length": 167.0, + "completions/min_terminated_length": 167.0, + "epoch": 0.5216352308611419, + "grad_norm": 2.2895874977111816, + "kl": 8.375, + "learning_rate": 5.946912200977794e-07, + "loss": 0.4919, + "num_tokens": 855108840.0, + "reward": 1.73876953125, + "reward_std": 0.5946817398071289, + "rewards/accuracy_reward/mean": 0.06653226166963577, + "rewards/accuracy_reward/std": 0.2494617998600006, + "rewards/format_reward/mean": 0.78515625, + "rewards/format_reward/std": 0.4111155867576599, + "rewards/tag_count_reward/mean": 0.88916015625, + "rewards/tag_count_reward/std": 0.23625560104846954, + "step": 1528 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.064453125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 914.890625, + "completions/mean_terminated_length": 836.8267211914062, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "epoch": 0.5219766151745328, + "grad_norm": 1.369377851486206, + "kl": 8.4140625, + "learning_rate": 5.941577309782441e-07, + "loss": 0.5102, + "num_tokens": 855659632.0, + "reward": 1.7109375, + "reward_std": 0.6631219983100891, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.775390625, + "rewards/format_reward/std": 0.41773295402526855, + "rewards/tag_count_reward/mean": 0.87890625, + "rewards/tag_count_reward/std": 0.24377650022506714, + "step": 1529 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1881.0, + "completions/mean_length": 908.5625, + "completions/mean_terminated_length": 842.6445922851562, + "completions/min_length": 161.0, + "completions/min_terminated_length": 161.0, + "epoch": 0.5223179994879236, + "grad_norm": 2.3144214153289795, + "kl": 7.3203125, + "learning_rate": 5.936241791848863e-07, + "loss": 0.4573, + "num_tokens": 856205616.0, + "reward": 1.7529296875, + "reward_std": 0.644518256187439, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.787109375, + "rewards/format_reward/std": 0.409751296043396, + "rewards/tag_count_reward/mean": 0.8916015625, + "rewards/tag_count_reward/std": 0.22331729531288147, + "step": 1530 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2003.0, + "completions/mean_length": 858.572265625, + "completions/mean_terminated_length": 779.277099609375, + "completions/min_length": 4.0, + "completions/min_terminated_length": 4.0, + "epoch": 0.5226593838013144, + "grad_norm": 1.9225261211395264, + "kl": 6.0625, + "learning_rate": 5.930905654749848e-07, + "loss": 0.4011, + "num_tokens": 856718485.0, + "reward": 1.78369140625, + "reward_std": 0.556646466255188, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.8046875, + "rewards/format_reward/std": 0.3968288004398346, + "rewards/tag_count_reward/mean": 0.90283203125, + "rewards/tag_count_reward/std": 0.21774683892726898, + "step": 1531 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2006.0, + "completions/mean_length": 847.63671875, + "completions/mean_terminated_length": 801.375244140625, + "completions/min_length": 75.0, + "completions/min_terminated_length": 75.0, + "epoch": 0.5230007681147051, + "grad_norm": 4.881642818450928, + "kl": 4.9609375, + "learning_rate": 5.925568906059073e-07, + "loss": 0.333, + "num_tokens": 857230027.0, + "reward": 1.8037109375, + "reward_std": 0.5452776551246643, + "rewards/accuracy_reward/mean": 0.037109375, + "rewards/accuracy_reward/std": 0.18921469151973724, + "rewards/format_reward/mean": 0.841796875, + "rewards/format_reward/std": 0.36528825759887695, + "rewards/tag_count_reward/mean": 0.9248046875, + "rewards/tag_count_reward/std": 0.1946192979812622, + "step": 1532 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.060546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2025.0, + "completions/mean_length": 944.017578125, + "completions/mean_terminated_length": 872.866943359375, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "epoch": 0.5233421524280959, + "grad_norm": 2.430119752883911, + "kl": 5.578125, + "learning_rate": 5.920231553351073e-07, + "loss": 0.3671, + "num_tokens": 857787924.0, + "reward": 1.78955078125, + "reward_std": 0.6080609560012817, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.798828125, + "rewards/format_reward/std": 0.4012683033943176, + "rewards/tag_count_reward/mean": 0.89697265625, + "rewards/tag_count_reward/std": 0.22449250519275665, + "step": 1533 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 893.072265625, + "completions/mean_terminated_length": 838.75048828125, + "completions/min_length": 166.0, + "completions/min_terminated_length": 166.0, + "epoch": 0.5236835367414867, + "grad_norm": 2.535127639770508, + "kl": 4.87109375, + "learning_rate": 5.914893604201244e-07, + "loss": 0.3157, + "num_tokens": 858329081.0, + "reward": 1.82861328125, + "reward_std": 0.5903012752532959, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.814453125, + "rewards/format_reward/std": 0.38912075757980347, + "rewards/tag_count_reward/mean": 0.90869140625, + "rewards/tag_count_reward/std": 0.2106221467256546, + "step": 1534 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1949.0, + "completions/mean_length": 877.5078125, + "completions/mean_terminated_length": 819.9425659179688, + "completions/min_length": 146.0, + "completions/min_terminated_length": 146.0, + "epoch": 0.5240249210548775, + "grad_norm": 1.4743084907531738, + "kl": 5.9609375, + "learning_rate": 5.909555066185829e-07, + "loss": 0.3752, + "num_tokens": 858867597.0, + "reward": 1.82568359375, + "reward_std": 0.5775442719459534, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.8203125, + "rewards/format_reward/std": 0.38430243730545044, + "rewards/tag_count_reward/mean": 0.91552734375, + "rewards/tag_count_reward/std": 0.2070624828338623, + "step": 1535 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2014.0, + "completions/mean_length": 856.703125, + "completions/mean_terminated_length": 813.2955932617188, + "completions/min_length": 130.0, + "completions/min_terminated_length": 130.0, + "epoch": 0.5243663053682683, + "grad_norm": 0.9794783592224121, + "kl": 6.03125, + "learning_rate": 5.904215946881907e-07, + "loss": 0.3721, + "num_tokens": 859384805.0, + "reward": 1.81201171875, + "reward_std": 0.5127238035202026, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.92138671875, + "rewards/tag_count_reward/std": 0.19733208417892456, + "step": 1536 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 924.396484375, + "completions/mean_terminated_length": 864.2860107421875, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "epoch": 0.5247076896816592, + "grad_norm": 3.236016035079956, + "kl": 8.375, + "learning_rate": 5.898876253867379e-07, + "loss": 0.4561, + "num_tokens": 859937536.0, + "reward": 1.701171875, + "reward_std": 0.6078129410743713, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.7734375, + "rewards/format_reward/std": 0.4190165400505066, + "rewards/tag_count_reward/mean": 0.888671875, + "rewards/tag_count_reward/std": 0.23263275623321533, + "step": 1537 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2029.0, + "completions/mean_length": 890.083984375, + "completions/mean_terminated_length": 828.1378173828125, + "completions/min_length": 216.0, + "completions/min_terminated_length": 216.0, + "epoch": 0.52504907399505, + "grad_norm": 4.2210469245910645, + "kl": 7.7421875, + "learning_rate": 5.893535994720965e-07, + "loss": 0.4208, + "num_tokens": 860468763.0, + "reward": 1.76806640625, + "reward_std": 0.6072068214416504, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.794921875, + "rewards/format_reward/std": 0.4041535556316376, + "rewards/tag_count_reward/mean": 0.90478515625, + "rewards/tag_count_reward/std": 0.21748776733875275, + "step": 1538 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1937.0, + "completions/mean_length": 863.265625, + "completions/mean_terminated_length": 810.0734252929688, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "epoch": 0.5253904583084408, + "grad_norm": 2.282278060913086, + "kl": 6.46875, + "learning_rate": 5.888195177022185e-07, + "loss": 0.356, + "num_tokens": 860986867.0, + "reward": 1.8134765625, + "reward_std": 0.5585914850234985, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.82421875, + "rewards/format_reward/std": 0.3810062110424042, + "rewards/tag_count_reward/mean": 0.9169921875, + "rewards/tag_count_reward/std": 0.19707830250263214, + "step": 1539 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2023.0, + "completions/mean_length": 837.60546875, + "completions/mean_terminated_length": 767.5826416015625, + "completions/min_length": 167.0, + "completions/min_terminated_length": 167.0, + "epoch": 0.5257318426218315, + "grad_norm": 2.910249710083008, + "kl": 8.0078125, + "learning_rate": 5.882853808351354e-07, + "loss": 0.4439, + "num_tokens": 861493369.0, + "reward": 1.80908203125, + "reward_std": 0.6293829679489136, + "rewards/accuracy_reward/mean": 0.099609375, + "rewards/accuracy_reward/std": 0.29977133870124817, + "rewards/format_reward/mean": 0.802734375, + "rewards/format_reward/std": 0.3983237147331238, + "rewards/tag_count_reward/mean": 0.90673828125, + "rewards/tag_count_reward/std": 0.21437686681747437, + "step": 1540 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2041.0, + "completions/mean_length": 952.8359375, + "completions/mean_terminated_length": 901.3251342773438, + "completions/min_length": 240.0, + "completions/min_terminated_length": 240.0, + "epoch": 0.5260732269352223, + "grad_norm": 1.2507978677749634, + "kl": 7.078125, + "learning_rate": 5.877511896289566e-07, + "loss": 0.4122, + "num_tokens": 862054181.0, + "reward": 1.79638671875, + "reward_std": 0.6029015779495239, + "rewards/accuracy_reward/mean": 0.08467742055654526, + "rewards/accuracy_reward/std": 0.278682142496109, + "rewards/format_reward/mean": 0.8046875, + "rewards/format_reward/std": 0.3968288004398346, + "rewards/tag_count_reward/mean": 0.90966796875, + "rewards/tag_count_reward/std": 0.21104364097118378, + "step": 1541 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2020.0, + "completions/mean_length": 869.470703125, + "completions/mean_terminated_length": 824.0507202148438, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "epoch": 0.5264146112486131, + "grad_norm": 2.267066717147827, + "kl": 6.5625, + "learning_rate": 5.872169448418688e-07, + "loss": 0.4339, + "num_tokens": 862573462.0, + "reward": 1.7919921875, + "reward_std": 0.5768818855285645, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.822265625, + "rewards/format_reward/std": 0.3826628625392914, + "rewards/tag_count_reward/mean": 0.9169921875, + "rewards/tag_count_reward/std": 0.20379072427749634, + "step": 1542 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1937.0, + "completions/mean_length": 877.001953125, + "completions/mean_terminated_length": 816.88916015625, + "completions/min_length": 139.0, + "completions/min_terminated_length": 139.0, + "epoch": 0.5267559955620039, + "grad_norm": 1.2301084995269775, + "kl": 6.296875, + "learning_rate": 5.866826472321351e-07, + "loss": 0.414, + "num_tokens": 863109975.0, + "reward": 1.78076171875, + "reward_std": 0.5611857175827026, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.8125, + "rewards/format_reward/std": 0.39069411158561707, + "rewards/tag_count_reward/mean": 0.91357421875, + "rewards/tag_count_reward/std": 0.20266404747962952, + "step": 1543 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2000.0, + "completions/mean_length": 915.37890625, + "completions/mean_terminated_length": 871.7281494140625, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "epoch": 0.5270973798753947, + "grad_norm": 1.3303993940353394, + "kl": 5.84375, + "learning_rate": 5.861482975580928e-07, + "loss": 0.3635, + "num_tokens": 863657049.0, + "reward": 1.724609375, + "reward_std": 0.5845118165016174, + "rewards/accuracy_reward/mean": 0.037109375, + "rewards/accuracy_reward/std": 0.18921469151973724, + "rewards/format_reward/mean": 0.78515625, + "rewards/format_reward/std": 0.4111155867576599, + "rewards/tag_count_reward/mean": 0.90234375, + "rewards/tag_count_reward/std": 0.21269488334655762, + "step": 1544 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 912.099609375, + "completions/mean_terminated_length": 856.235595703125, + "completions/min_length": 108.0, + "completions/min_terminated_length": 108.0, + "epoch": 0.5274387641887855, + "grad_norm": 2.149855375289917, + "kl": 6.6015625, + "learning_rate": 5.856138965781538e-07, + "loss": 0.373, + "num_tokens": 864208796.0, + "reward": 1.7548828125, + "reward_std": 0.5656696557998657, + "rewards/accuracy_reward/mean": 0.03515625, + "rewards/accuracy_reward/std": 0.1843547374010086, + "rewards/format_reward/mean": 0.80859375, + "rewards/format_reward/std": 0.3937928080558777, + "rewards/tag_count_reward/mean": 0.9111328125, + "rewards/tag_count_reward/std": 0.20129980146884918, + "step": 1545 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2012.0, + "completions/mean_length": 908.794921875, + "completions/mean_terminated_length": 845.3753051757812, + "completions/min_length": 35.0, + "completions/min_terminated_length": 35.0, + "epoch": 0.5277801485021764, + "grad_norm": 1.1292212009429932, + "kl": 7.1015625, + "learning_rate": 5.850794450508026e-07, + "loss": 0.4205, + "num_tokens": 864750387.0, + "reward": 1.69970703125, + "reward_std": 0.5838012099266052, + "rewards/accuracy_reward/mean": 0.03333333507180214, + "rewards/accuracy_reward/std": 0.17969276010990143, + "rewards/format_reward/mean": 0.7734375, + "rewards/format_reward/std": 0.4190165400505066, + "rewards/tag_count_reward/mean": 0.89501953125, + "rewards/tag_count_reward/std": 0.22138507664203644, + "step": 1546 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2001.0, + "completions/mean_length": 991.205078125, + "completions/mean_terminated_length": 946.0061645507812, + "completions/min_length": 153.0, + "completions/min_terminated_length": 153.0, + "epoch": 0.5281215328155672, + "grad_norm": 0.747988224029541, + "kl": 6.203125, + "learning_rate": 5.845449437345952e-07, + "loss": 0.3744, + "num_tokens": 865338316.0, + "reward": 1.7763671875, + "reward_std": 0.546890139579773, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.8203125, + "rewards/format_reward/std": 0.38430243730545044, + "rewards/tag_count_reward/mean": 0.9150390625, + "rewards/tag_count_reward/std": 0.20055793225765228, + "step": 1547 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2012.0, + "completions/mean_length": 862.59765625, + "completions/mean_terminated_length": 821.8869018554688, + "completions/min_length": 166.0, + "completions/min_terminated_length": 166.0, + "epoch": 0.5284629171289579, + "grad_norm": 1.1407604217529297, + "kl": 5.671875, + "learning_rate": 5.840103933881584e-07, + "loss": 0.3333, + "num_tokens": 865862510.0, + "reward": 1.79541015625, + "reward_std": 0.5912949442863464, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.8046875, + "rewards/format_reward/std": 0.3968288004398346, + "rewards/tag_count_reward/mean": 0.92041015625, + "rewards/tag_count_reward/std": 0.18998514115810394, + "step": 1548 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 869.046875, + "completions/mean_terminated_length": 828.5576171875, + "completions/min_length": 189.0, + "completions/min_terminated_length": 189.0, + "epoch": 0.5288043014423487, + "grad_norm": 1.438033103942871, + "kl": 5.171875, + "learning_rate": 5.834757947701889e-07, + "loss": 0.298, + "num_tokens": 866384982.0, + "reward": 1.78955078125, + "reward_std": 0.5551824569702148, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.8125, + "rewards/format_reward/std": 0.39069411158561707, + "rewards/tag_count_reward/mean": 0.92236328125, + "rewards/tag_count_reward/std": 0.18427114188671112, + "step": 1549 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1937.0, + "completions/mean_length": 913.712890625, + "completions/mean_terminated_length": 848.0929565429688, + "completions/min_length": 148.0, + "completions/min_terminated_length": 148.0, + "epoch": 0.5291456857557395, + "grad_norm": 1.2410811185836792, + "kl": 7.703125, + "learning_rate": 5.829411486394516e-07, + "loss": 0.4634, + "num_tokens": 866937091.0, + "reward": 1.69873046875, + "reward_std": 0.6290674805641174, + "rewards/accuracy_reward/mean": 0.05645161122083664, + "rewards/accuracy_reward/std": 0.23102474212646484, + "rewards/format_reward/mean": 0.755859375, + "rewards/format_reward/std": 0.42999663949012756, + "rewards/tag_count_reward/mean": 0.88818359375, + "rewards/tag_count_reward/std": 0.2257234901189804, + "step": 1550 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1924.0, + "completions/mean_length": 904.966796875, + "completions/mean_terminated_length": 856.0794677734375, + "completions/min_length": 189.0, + "completions/min_terminated_length": 189.0, + "epoch": 0.5294870700691303, + "grad_norm": 2.584840774536133, + "kl": 5.7578125, + "learning_rate": 5.824064557547785e-07, + "loss": 0.3879, + "num_tokens": 867483090.0, + "reward": 1.78076171875, + "reward_std": 0.5492658615112305, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.806640625, + "rewards/format_reward/std": 0.39531853795051575, + "rewards/tag_count_reward/mean": 0.90966796875, + "rewards/tag_count_reward/std": 0.20694707334041595, + "step": 1551 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 816.796875, + "completions/mean_terminated_length": 782.1846923828125, + "completions/min_length": 164.0, + "completions/min_terminated_length": 164.0, + "epoch": 0.5298284543825211, + "grad_norm": 1.3334747552871704, + "kl": 6.078125, + "learning_rate": 5.81871716875069e-07, + "loss": 0.3443, + "num_tokens": 867971066.0, + "reward": 1.8603515625, + "reward_std": 0.6013159155845642, + "rewards/accuracy_reward/mean": 0.111328125, + "rewards/accuracy_reward/std": 0.31484565138816833, + "rewards/format_reward/mean": 0.828125, + "rewards/format_reward/std": 0.3776407241821289, + "rewards/tag_count_reward/mean": 0.9208984375, + "rewards/tag_count_reward/std": 0.19242700934410095, + "step": 1552 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1987.0, + "completions/mean_length": 867.712890625, + "completions/mean_terminated_length": 819.7337036132812, + "completions/min_length": 140.0, + "completions/min_terminated_length": 140.0, + "epoch": 0.5301698386959119, + "grad_norm": 1.3950985670089722, + "kl": 7.46875, + "learning_rate": 5.813369327592867e-07, + "loss": 0.4191, + "num_tokens": 868499111.0, + "reward": 1.67822265625, + "reward_std": 0.6357830762863159, + "rewards/accuracy_reward/mean": 0.025390625, + "rewards/accuracy_reward/std": 0.15746226906776428, + "rewards/format_reward/mean": 0.765625, + "rewards/format_reward/std": 0.42402184009552, + "rewards/tag_count_reward/mean": 0.88720703125, + "rewards/tag_count_reward/std": 0.2257785201072693, + "step": 1553 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2029.0, + "completions/mean_length": 847.892578125, + "completions/mean_terminated_length": 799.107666015625, + "completions/min_length": 134.0, + "completions/min_terminated_length": 134.0, + "epoch": 0.5305112230093028, + "grad_norm": 3.1379051208496094, + "kl": 6.96875, + "learning_rate": 5.808021041664599e-07, + "loss": 0.3617, + "num_tokens": 869014336.0, + "reward": 1.73681640625, + "reward_std": 0.5985440611839294, + "rewards/accuracy_reward/mean": 0.05443548411130905, + "rewards/accuracy_reward/std": 0.227104052901268, + "rewards/format_reward/mean": 0.77734375, + "rewards/format_reward/std": 0.41643625497817993, + "rewards/tag_count_reward/mean": 0.90673828125, + "rewards/tag_count_reward/std": 0.20623478293418884, + "step": 1554 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2013.0, + "completions/mean_length": 850.486328125, + "completions/mean_terminated_length": 791.5921630859375, + "completions/min_length": 234.0, + "completions/min_terminated_length": 234.0, + "epoch": 0.5308526073226936, + "grad_norm": 1.0754282474517822, + "kl": 7.5703125, + "learning_rate": 5.802672318556802e-07, + "loss": 0.451, + "num_tokens": 869526233.0, + "reward": 1.76708984375, + "reward_std": 0.589768648147583, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.802734375, + "rewards/format_reward/std": 0.3983237147331238, + "rewards/tag_count_reward/mean": 0.90380859375, + "rewards/tag_count_reward/std": 0.21874071657657623, + "step": 1555 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 919.279296875, + "completions/mean_terminated_length": 849.0270385742188, + "completions/min_length": 119.0, + "completions/min_terminated_length": 119.0, + "epoch": 0.5311939916360843, + "grad_norm": 3.4173245429992676, + "kl": 7.9765625, + "learning_rate": 5.797323165861007e-07, + "loss": 0.4163, + "num_tokens": 870070072.0, + "reward": 1.7373046875, + "reward_std": 0.6712255477905273, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.759765625, + "rewards/format_reward/std": 0.4276435375213623, + "rewards/tag_count_reward/mean": 0.8798828125, + "rewards/tag_count_reward/std": 0.23611247539520264, + "step": 1556 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2007.0, + "completions/mean_length": 854.3359375, + "completions/mean_terminated_length": 803.2831420898438, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "epoch": 0.5315353759494751, + "grad_norm": 1.6247764825820923, + "kl": 7.0859375, + "learning_rate": 5.791973591169359e-07, + "loss": 0.4521, + "num_tokens": 870587988.0, + "reward": 1.76611328125, + "reward_std": 0.6064735651016235, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.80078125, + "rewards/format_reward/std": 0.39980348944664, + "rewards/tag_count_reward/mean": 0.90283203125, + "rewards/tag_count_reward/std": 0.21774683892726898, + "step": 1557 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1935.0, + "completions/mean_length": 899.740234375, + "completions/mean_terminated_length": 848.1856689453125, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "epoch": 0.5318767602628659, + "grad_norm": 1.4438574314117432, + "kl": 6.03125, + "learning_rate": 5.786623602074602e-07, + "loss": 0.3463, + "num_tokens": 871129135.0, + "reward": 1.767578125, + "reward_std": 0.6126869916915894, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.787109375, + "rewards/format_reward/std": 0.409751296043396, + "rewards/tag_count_reward/mean": 0.90234375, + "rewards/tag_count_reward/std": 0.21724654734134674, + "step": 1558 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 903.208984375, + "completions/mean_terminated_length": 856.6727294921875, + "completions/min_length": 180.0, + "completions/min_terminated_length": 180.0, + "epoch": 0.5322181445762567, + "grad_norm": 2.136967658996582, + "kl": 5.21484375, + "learning_rate": 5.781273206170065e-07, + "loss": 0.311, + "num_tokens": 871661450.0, + "reward": 1.76416015625, + "reward_std": 0.589134693145752, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.79296875, + "rewards/format_reward/std": 0.40557438135147095, + "rewards/tag_count_reward/mean": 0.91650390625, + "rewards/tag_count_reward/std": 0.19531220197677612, + "step": 1559 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1958.0, + "completions/mean_length": 860.767578125, + "completions/mean_terminated_length": 841.9226684570312, + "completions/min_length": 141.0, + "completions/min_terminated_length": 141.0, + "epoch": 0.5325595288896475, + "grad_norm": 2.0706610679626465, + "kl": 4.4453125, + "learning_rate": 5.775922411049657e-07, + "loss": 0.2632, + "num_tokens": 872184963.0, + "reward": 1.82861328125, + "reward_std": 0.5420112013816833, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.822265625, + "rewards/format_reward/std": 0.3826628625392914, + "rewards/tag_count_reward/mean": 0.92431640625, + "rewards/tag_count_reward/std": 0.18174928426742554, + "step": 1560 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 942.40234375, + "completions/mean_terminated_length": 892.7632446289062, + "completions/min_length": 133.0, + "completions/min_terminated_length": 133.0, + "epoch": 0.5329009132030383, + "grad_norm": 2.4473965167999268, + "kl": 6.07421875, + "learning_rate": 5.770571224307855e-07, + "loss": 0.3488, + "num_tokens": 872756385.0, + "reward": 1.701171875, + "reward_std": 0.6234464049339294, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.763671875, + "rewards/format_reward/std": 0.42524150013923645, + "rewards/tag_count_reward/mean": 0.890625, + "rewards/tag_count_reward/std": 0.21215508878231049, + "step": 1561 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1909.0, + "completions/mean_length": 802.51171875, + "completions/mean_terminated_length": 775.1656494140625, + "completions/min_length": 18.0, + "completions/min_terminated_length": 18.0, + "epoch": 0.5332422975164292, + "grad_norm": 2.3419058322906494, + "kl": 5.203125, + "learning_rate": 5.765219653539687e-07, + "loss": 0.3138, + "num_tokens": 873240055.0, + "reward": 1.82763671875, + "reward_std": 0.6118506193161011, + "rewards/accuracy_reward/mean": 0.09879032522439957, + "rewards/accuracy_reward/std": 0.2986815273761749, + "rewards/format_reward/mean": 0.8203125, + "rewards/format_reward/std": 0.38430243730545044, + "rewards/tag_count_reward/mean": 0.91162109375, + "rewards/tag_count_reward/std": 0.20060259103775024, + "step": 1562 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 859.98828125, + "completions/mean_terminated_length": 806.64892578125, + "completions/min_length": 19.0, + "completions/min_terminated_length": 19.0, + "epoch": 0.53358368182982, + "grad_norm": 2.196070909500122, + "kl": 5.71875, + "learning_rate": 5.759867706340731e-07, + "loss": 0.3871, + "num_tokens": 873755809.0, + "reward": 1.79052734375, + "reward_std": 0.5443712472915649, + "rewards/accuracy_reward/mean": 0.04233871027827263, + "rewards/accuracy_reward/std": 0.2015640139579773, + "rewards/format_reward/mean": 0.830078125, + "rewards/format_reward/std": 0.3759314715862274, + "rewards/tag_count_reward/mean": 0.91943359375, + "rewards/tag_count_reward/std": 0.19778190553188324, + "step": 1563 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 894.728515625, + "completions/mean_terminated_length": 828.0103149414062, + "completions/min_length": 114.0, + "completions/min_terminated_length": 114.0, + "epoch": 0.5339250661432107, + "grad_norm": 3.4765467643737793, + "kl": 6.921875, + "learning_rate": 5.754515390307095e-07, + "loss": 0.357, + "num_tokens": 874297702.0, + "reward": 1.71923828125, + "reward_std": 0.648097038269043, + "rewards/accuracy_reward/mean": 0.09677419066429138, + "rewards/accuracy_reward/std": 0.2959485352039337, + "rewards/format_reward/mean": 0.74609375, + "rewards/format_reward/std": 0.43567025661468506, + "rewards/tag_count_reward/mean": 0.87939453125, + "rewards/tag_count_reward/std": 0.22224663197994232, + "step": 1564 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1970.0, + "completions/mean_length": 854.525390625, + "completions/mean_terminated_length": 813.5374145507812, + "completions/min_length": 172.0, + "completions/min_terminated_length": 172.0, + "epoch": 0.5342664504566015, + "grad_norm": 4.09050989151001, + "kl": 6.34375, + "learning_rate": 5.749162713035415e-07, + "loss": 0.3373, + "num_tokens": 874815875.0, + "reward": 1.7197265625, + "reward_std": 0.5855885744094849, + "rewards/accuracy_reward/mean": 0.037109375, + "rewards/accuracy_reward/std": 0.18921469151973724, + "rewards/format_reward/mean": 0.78125, + "rewards/format_reward/std": 0.41380295157432556, + "rewards/tag_count_reward/mean": 0.9013671875, + "rewards/tag_count_reward/std": 0.19792981445789337, + "step": 1565 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1964.0, + "completions/mean_length": 879.75, + "completions/mean_terminated_length": 824.8016357421875, + "completions/min_length": 143.0, + "completions/min_terminated_length": 143.0, + "epoch": 0.5346078347699923, + "grad_norm": 3.7021701335906982, + "kl": 6.5078125, + "learning_rate": 5.743809682122836e-07, + "loss": 0.3449, + "num_tokens": 875343107.0, + "reward": 1.7158203125, + "reward_std": 0.6114732027053833, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.7734375, + "rewards/format_reward/std": 0.4190165400505066, + "rewards/tag_count_reward/mean": 0.8994140625, + "rewards/tag_count_reward/std": 0.19632047414779663, + "step": 1566 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1967.0, + "completions/mean_length": 848.44140625, + "completions/mean_terminated_length": 812.2373657226562, + "completions/min_length": 5.0, + "completions/min_terminated_length": 5.0, + "epoch": 0.5349492190833831, + "grad_norm": 4.1037702560424805, + "kl": 5.703125, + "learning_rate": 5.738456305167007e-07, + "loss": 0.3173, + "num_tokens": 875859541.0, + "reward": 1.763671875, + "reward_std": 0.6081827878952026, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.794921875, + "rewards/format_reward/std": 0.4041535556316376, + "rewards/tag_count_reward/mean": 0.892578125, + "rewards/tag_count_reward/std": 0.2210400402545929, + "step": 1567 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1989.0, + "completions/mean_length": 808.1171875, + "completions/mean_terminated_length": 762.9392700195312, + "completions/min_length": 141.0, + "completions/min_terminated_length": 141.0, + "epoch": 0.5352906033967739, + "grad_norm": 1.329008936882019, + "kl": 5.921875, + "learning_rate": 5.733102589766068e-07, + "loss": 0.3459, + "num_tokens": 876348289.0, + "reward": 1.8056640625, + "reward_std": 0.6167274117469788, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.80078125, + "rewards/format_reward/std": 0.39980348944664, + "rewards/tag_count_reward/mean": 0.9130859375, + "rewards/tag_count_reward/std": 0.19349662959575653, + "step": 1568 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2020.0, + "completions/mean_length": 885.88671875, + "completions/mean_terminated_length": 836.183349609375, + "completions/min_length": 135.0, + "completions/min_terminated_length": 135.0, + "epoch": 0.5356319877101647, + "grad_norm": 1.266638159751892, + "kl": 5.7734375, + "learning_rate": 5.727748543518637e-07, + "loss": 0.3505, + "num_tokens": 876871111.0, + "reward": 1.81103515625, + "reward_std": 0.5950206518173218, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.82421875, + "rewards/format_reward/std": 0.3810062110424042, + "rewards/tag_count_reward/mean": 0.90673828125, + "rewards/tag_count_reward/std": 0.20682698488235474, + "step": 1569 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2024.0, + "completions/mean_length": 885.673828125, + "completions/mean_terminated_length": 845.755615234375, + "completions/min_length": 187.0, + "completions/min_terminated_length": 187.0, + "epoch": 0.5359733720235555, + "grad_norm": 1.0821884870529175, + "kl": 5.4140625, + "learning_rate": 5.722394174023805e-07, + "loss": 0.3236, + "num_tokens": 877401328.0, + "reward": 1.78466796875, + "reward_std": 0.5873010158538818, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.806640625, + "rewards/format_reward/std": 0.39531853795051575, + "rewards/tag_count_reward/mean": 0.91552734375, + "rewards/tag_count_reward/std": 0.19800402224063873, + "step": 1570 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2018.0, + "completions/mean_length": 856.93359375, + "completions/mean_terminated_length": 813.534423828125, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.5363147563369464, + "grad_norm": 1.3648273944854736, + "kl": 5.0078125, + "learning_rate": 5.717039488881118e-07, + "loss": 0.2963, + "num_tokens": 877919006.0, + "reward": 1.833984375, + "reward_std": 0.602922797203064, + "rewards/accuracy_reward/mean": 0.1015625, + "rewards/accuracy_reward/std": 0.30236753821372986, + "rewards/format_reward/mean": 0.81640625, + "rewards/format_reward/std": 0.3875311613082886, + "rewards/tag_count_reward/mean": 0.916015625, + "rewards/tag_count_reward/std": 0.19604040682315826, + "step": 1571 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1970.0, + "completions/mean_length": 790.580078125, + "completions/mean_terminated_length": 742.11962890625, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "epoch": 0.5366561406503371, + "grad_norm": 2.4586451053619385, + "kl": 5.109375, + "learning_rate": 5.711684495690573e-07, + "loss": 0.3362, + "num_tokens": 878399975.0, + "reward": 1.8408203125, + "reward_std": 0.5516421794891357, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.9228515625, + "rewards/tag_count_reward/std": 0.18873685598373413, + "step": 1572 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2014.0, + "completions/mean_length": 891.001953125, + "completions/mean_terminated_length": 839.0550537109375, + "completions/min_length": 150.0, + "completions/min_terminated_length": 150.0, + "epoch": 0.5369975249637279, + "grad_norm": 1.2200103998184204, + "kl": 4.88671875, + "learning_rate": 5.706329202052605e-07, + "loss": 0.3001, + "num_tokens": 878926104.0, + "reward": 1.7900390625, + "reward_std": 0.5616840124130249, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.814453125, + "rewards/format_reward/std": 0.38912075757980347, + "rewards/tag_count_reward/mean": 0.9169921875, + "rewards/tag_count_reward/std": 0.19204923510551453, + "step": 1573 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1987.0, + "completions/mean_length": 920.75, + "completions/mean_terminated_length": 855.5371704101562, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "epoch": 0.5373389092771187, + "grad_norm": 2.1945228576660156, + "kl": 6.25390625, + "learning_rate": 5.700973615568072e-07, + "loss": 0.3679, + "num_tokens": 879475128.0, + "reward": 1.72412109375, + "reward_std": 0.6320927739143372, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.779296875, + "rewards/format_reward/std": 0.4151262938976288, + "rewards/tag_count_reward/mean": 0.89599609375, + "rewards/tag_count_reward/std": 0.21626292169094086, + "step": 1574 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1978.0, + "completions/mean_length": 851.12890625, + "completions/mean_terminated_length": 819.9478759765625, + "completions/min_length": 178.0, + "completions/min_terminated_length": 178.0, + "epoch": 0.5376802935905095, + "grad_norm": 0.8738031983375549, + "kl": 4.16015625, + "learning_rate": 5.695617743838252e-07, + "loss": 0.2279, + "num_tokens": 879985882.0, + "reward": 1.81005859375, + "reward_std": 0.5117999315261841, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.83203125, + "rewards/format_reward/std": 0.374204158782959, + "rewards/tag_count_reward/mean": 0.93505859375, + "rewards/tag_count_reward/std": 0.16270823776721954, + "step": 1575 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 856.6484375, + "completions/mean_terminated_length": 813.2388916015625, + "completions/min_length": 59.0, + "completions/min_terminated_length": 59.0, + "epoch": 0.5380216779039003, + "grad_norm": 1.9138249158859253, + "kl": 4.5625, + "learning_rate": 5.690261594464824e-07, + "loss": 0.2513, + "num_tokens": 880506678.0, + "reward": 1.74462890625, + "reward_std": 0.4995850920677185, + "rewards/accuracy_reward/mean": 0.013671875, + "rewards/accuracy_reward/std": 0.1162383034825325, + "rewards/format_reward/mean": 0.81640625, + "rewards/format_reward/std": 0.3875311613082886, + "rewards/tag_count_reward/mean": 0.91455078125, + "rewards/tag_count_reward/std": 0.19446396827697754, + "step": 1576 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1986.0, + "completions/mean_length": 842.876953125, + "completions/mean_terminated_length": 786.1942749023438, + "completions/min_length": 146.0, + "completions/min_terminated_length": 146.0, + "epoch": 0.5383630622172911, + "grad_norm": 1.5206162929534912, + "kl": 5.23828125, + "learning_rate": 5.68490517504986e-07, + "loss": 0.327, + "num_tokens": 881008359.0, + "reward": 1.80322265625, + "reward_std": 0.5785903930664062, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.814453125, + "rewards/format_reward/std": 0.38912075757980347, + "rewards/tag_count_reward/mean": 0.91064453125, + "rewards/tag_count_reward/std": 0.20618844032287598, + "step": 1577 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2029.0, + "completions/mean_length": 905.759765625, + "completions/mean_terminated_length": 868.9132690429688, + "completions/min_length": 145.0, + "completions/min_terminated_length": 145.0, + "epoch": 0.538704446530682, + "grad_norm": 1.0344030857086182, + "kl": 4.80078125, + "learning_rate": 5.67954849319582e-07, + "loss": 0.2771, + "num_tokens": 881549436.0, + "reward": 1.78466796875, + "reward_std": 0.539876401424408, + "rewards/accuracy_reward/mean": 0.03515625, + "rewards/accuracy_reward/std": 0.1843547374010086, + "rewards/format_reward/mean": 0.828125, + "rewards/format_reward/std": 0.3776407241821289, + "rewards/tag_count_reward/mean": 0.92138671875, + "rewards/tag_count_reward/std": 0.19167259335517883, + "step": 1578 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1861.0, + "completions/mean_length": 857.669921875, + "completions/mean_terminated_length": 829.10205078125, + "completions/min_length": 142.0, + "completions/min_terminated_length": 142.0, + "epoch": 0.5390458308440728, + "grad_norm": 3.1057684421539307, + "kl": 4.00390625, + "learning_rate": 5.674191556505533e-07, + "loss": 0.2752, + "num_tokens": 882069283.0, + "reward": 1.8603515625, + "reward_std": 0.5147795677185059, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.9404296875, + "rewards/tag_count_reward/std": 0.1606195569038391, + "step": 1579 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2019.0, + "completions/mean_length": 868.314453125, + "completions/mean_terminated_length": 825.3299560546875, + "completions/min_length": 162.0, + "completions/min_terminated_length": 162.0, + "epoch": 0.5393872151574635, + "grad_norm": 1.5593984127044678, + "kl": 5.58203125, + "learning_rate": 5.668834372582195e-07, + "loss": 0.3297, + "num_tokens": 882602084.0, + "reward": 1.7783203125, + "reward_std": 0.5504990816116333, + "rewards/accuracy_reward/mean": 0.037109375, + "rewards/accuracy_reward/std": 0.18921469151973724, + "rewards/format_reward/mean": 0.82421875, + "rewards/format_reward/std": 0.3810062110424042, + "rewards/tag_count_reward/mean": 0.9169921875, + "rewards/tag_count_reward/std": 0.19395042955875397, + "step": 1580 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1879.0, + "completions/mean_length": 876.46875, + "completions/mean_terminated_length": 831.3184204101562, + "completions/min_length": 113.0, + "completions/min_terminated_length": 113.0, + "epoch": 0.5397285994708543, + "grad_norm": 1.1250920295715332, + "kl": 5.21875, + "learning_rate": 5.663476949029342e-07, + "loss": 0.3066, + "num_tokens": 883135796.0, + "reward": 1.77001953125, + "reward_std": 0.5458778738975525, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.80859375, + "rewards/format_reward/std": 0.3937928080558777, + "rewards/tag_count_reward/mean": 0.91064453125, + "rewards/tag_count_reward/std": 0.19894284009933472, + "step": 1581 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1998.0, + "completions/mean_length": 795.607421875, + "completions/mean_terminated_length": 768.1098022460938, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "epoch": 0.5400699837842451, + "grad_norm": 1.7664976119995117, + "kl": 4.20703125, + "learning_rate": 5.65811929345086e-07, + "loss": 0.2241, + "num_tokens": 883626971.0, + "reward": 1.8818359375, + "reward_std": 0.5036077499389648, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.849609375, + "rewards/format_reward/std": 0.35780346393585205, + "rewards/tag_count_reward/mean": 0.9267578125, + "rewards/tag_count_reward/std": 0.192208394408226, + "step": 1582 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1958.0, + "completions/mean_length": 846.677734375, + "completions/mean_terminated_length": 805.4202270507812, + "completions/min_length": 158.0, + "completions/min_terminated_length": 158.0, + "epoch": 0.5404113680976359, + "grad_norm": 1.2929701805114746, + "kl": 4.62890625, + "learning_rate": 5.652761413450965e-07, + "loss": 0.2653, + "num_tokens": 884126726.0, + "reward": 1.8408203125, + "reward_std": 0.4792023003101349, + "rewards/accuracy_reward/mean": 0.033203125, + "rewards/accuracy_reward/std": 0.17934183776378632, + "rewards/format_reward/mean": 0.8671875, + "rewards/format_reward/std": 0.33970388770103455, + "rewards/tag_count_reward/mean": 0.9404296875, + "rewards/tag_count_reward/std": 0.1666000783443451, + "step": 1583 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 820.994140625, + "completions/mean_terminated_length": 781.4132690429688, + "completions/min_length": 5.0, + "completions/min_terminated_length": 5.0, + "epoch": 0.5407527524110267, + "grad_norm": 3.424382209777832, + "kl": 4.8671875, + "learning_rate": 5.647403316634181e-07, + "loss": 0.3484, + "num_tokens": 884621955.0, + "reward": 1.8779296875, + "reward_std": 0.48530155420303345, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.8828125, + "rewards/format_reward/std": 0.32195815443992615, + "rewards/tag_count_reward/mean": 0.9423828125, + "rewards/tag_count_reward/std": 0.1672869324684143, + "step": 1584 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2009.0, + "completions/mean_length": 845.580078125, + "completions/mean_terminated_length": 809.2897338867188, + "completions/min_length": 187.0, + "completions/min_terminated_length": 187.0, + "epoch": 0.5410941367244175, + "grad_norm": 0.9744515419006348, + "kl": 5.57421875, + "learning_rate": 5.64204501060535e-07, + "loss": 0.3258, + "num_tokens": 885142044.0, + "reward": 1.87158203125, + "reward_std": 0.5376770496368408, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.93408203125, + "rewards/tag_count_reward/std": 0.1841985285282135, + "step": 1585 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1858.0, + "completions/mean_length": 810.689453125, + "completions/mean_terminated_length": 768.1959838867188, + "completions/min_length": 226.0, + "completions/min_terminated_length": 226.0, + "epoch": 0.5414355210378083, + "grad_norm": 0.995830774307251, + "kl": 5.50390625, + "learning_rate": 5.636686502969606e-07, + "loss": 0.3226, + "num_tokens": 885636301.0, + "reward": 1.837890625, + "reward_std": 0.5120077133178711, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.841796875, + "rewards/format_reward/std": 0.36528825759887695, + "rewards/tag_count_reward/mean": 0.9296875, + "rewards/tag_count_reward/std": 0.18155530095100403, + "step": 1586 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1980.0, + "completions/mean_length": 823.3671875, + "completions/mean_terminated_length": 783.8628540039062, + "completions/min_length": 12.0, + "completions/min_terminated_length": 12.0, + "epoch": 0.5417769053511992, + "grad_norm": 1.4965887069702148, + "kl": 4.90234375, + "learning_rate": 5.631327801332373e-07, + "loss": 0.3153, + "num_tokens": 886141625.0, + "reward": 1.87939453125, + "reward_std": 0.5356603860855103, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.86328125, + "rewards/format_reward/std": 0.3438861668109894, + "rewards/tag_count_reward/mean": 0.93603515625, + "rewards/tag_count_reward/std": 0.17327652871608734, + "step": 1587 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2019.0, + "completions/mean_length": 846.3046875, + "completions/mean_terminated_length": 787.2048950195312, + "completions/min_length": 194.0, + "completions/min_terminated_length": 194.0, + "epoch": 0.5421182896645899, + "grad_norm": 1.4262303113937378, + "kl": 6.828125, + "learning_rate": 5.625968913299344e-07, + "loss": 0.4179, + "num_tokens": 886654533.0, + "reward": 1.83349609375, + "reward_std": 0.5428067445755005, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.841796875, + "rewards/format_reward/std": 0.36528825759887695, + "rewards/tag_count_reward/mean": 0.91943359375, + "rewards/tag_count_reward/std": 0.1952926218509674, + "step": 1588 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2014.0, + "completions/mean_length": 887.185546875, + "completions/mean_terminated_length": 847.3192138671875, + "completions/min_length": 176.0, + "completions/min_terminated_length": 176.0, + "epoch": 0.5424596739779807, + "grad_norm": 1.3533570766448975, + "kl": 5.72265625, + "learning_rate": 5.620609846476486e-07, + "loss": 0.334, + "num_tokens": 887184228.0, + "reward": 1.83447265625, + "reward_std": 0.4780813157558441, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.92822265625, + "rewards/tag_count_reward/std": 0.18597961962223053, + "step": 1589 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1926.0, + "completions/mean_length": 902.568359375, + "completions/mean_terminated_length": 836.3037109375, + "completions/min_length": 25.0, + "completions/min_terminated_length": 25.0, + "epoch": 0.5428010582913715, + "grad_norm": 1.906775712966919, + "kl": 6.6953125, + "learning_rate": 5.615250608470009e-07, + "loss": 0.3857, + "num_tokens": 887721623.0, + "reward": 1.7998046875, + "reward_std": 0.6019806861877441, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.798828125, + "rewards/format_reward/std": 0.4012683033943176, + "rewards/tag_count_reward/mean": 0.9072265625, + "rewards/tag_count_reward/std": 0.21143096685409546, + "step": 1590 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2031.0, + "completions/mean_length": 839.203125, + "completions/mean_terminated_length": 800.2096557617188, + "completions/min_length": 141.0, + "completions/min_terminated_length": 141.0, + "epoch": 0.5431424426047623, + "grad_norm": 1.6059449911117554, + "kl": 6.1484375, + "learning_rate": 5.609891206886373e-07, + "loss": 0.3718, + "num_tokens": 888228847.0, + "reward": 1.8369140625, + "reward_std": 0.5652753710746765, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.9248046875, + "rewards/tag_count_reward/std": 0.1939898282289505, + "step": 1591 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2031.0, + "completions/mean_length": 895.525390625, + "completions/mean_terminated_length": 843.7816162109375, + "completions/min_length": 62.0, + "completions/min_terminated_length": 62.0, + "epoch": 0.5434838269181531, + "grad_norm": 2.739230155944824, + "kl": 7.375, + "learning_rate": 5.604531649332267e-07, + "loss": 0.4237, + "num_tokens": 888770572.0, + "reward": 1.74609375, + "reward_std": 0.6087614893913269, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.791015625, + "rewards/format_reward/std": 0.40698084235191345, + "rewards/tag_count_reward/mean": 0.8984375, + "rewards/tag_count_reward/std": 0.21258702874183655, + "step": 1592 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 937.939453125, + "completions/mean_terminated_length": 880.954833984375, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "epoch": 0.5438252112315439, + "grad_norm": 1.3750349283218384, + "kl": 7.7890625, + "learning_rate": 5.599171943414605e-07, + "loss": 0.4701, + "num_tokens": 889328013.0, + "reward": 1.8046875, + "reward_std": 0.6562377214431763, + "rewards/accuracy_reward/mean": 0.11491935700178146, + "rewards/accuracy_reward/std": 0.3192465901374817, + "rewards/format_reward/mean": 0.796875, + "rewards/format_reward/std": 0.4027182459831238, + "rewards/tag_count_reward/mean": 0.896484375, + "rewards/tag_count_reward/std": 0.22124744951725006, + "step": 1593 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1991.0, + "completions/mean_length": 888.693359375, + "completions/mean_terminated_length": 829.1807250976562, + "completions/min_length": 40.0, + "completions/min_terminated_length": 40.0, + "epoch": 0.5441665955449347, + "grad_norm": 1.3322317600250244, + "kl": 5.6015625, + "learning_rate": 5.593812096740507e-07, + "loss": 0.3468, + "num_tokens": 889863520.0, + "reward": 1.83154296875, + "reward_std": 0.5643051266670227, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.82421875, + "rewards/format_reward/std": 0.3810062110424042, + "rewards/tag_count_reward/mean": 0.91552734375, + "rewards/tag_count_reward/std": 0.20408765971660614, + "step": 1594 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 879.61328125, + "completions/mean_terminated_length": 814.569091796875, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "epoch": 0.5445079798583256, + "grad_norm": 1.2477127313613892, + "kl": 6.1640625, + "learning_rate": 5.588452116917299e-07, + "loss": 0.4084, + "num_tokens": 890394282.0, + "reward": 1.787109375, + "reward_std": 0.5605219602584839, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.8203125, + "rewards/format_reward/std": 0.38430243730545044, + "rewards/tag_count_reward/mean": 0.900390625, + "rewards/tag_count_reward/std": 0.22358457744121552, + "step": 1595 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1921.0, + "completions/mean_length": 807.458984375, + "completions/mean_terminated_length": 770.01806640625, + "completions/min_length": 121.0, + "completions/min_terminated_length": 121.0, + "epoch": 0.5448493641717163, + "grad_norm": 2.605314254760742, + "kl": 4.890625, + "learning_rate": 5.583092011552487e-07, + "loss": 0.3089, + "num_tokens": 890893653.0, + "reward": 1.7900390625, + "reward_std": 0.5496975183486938, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.8203125, + "rewards/format_reward/std": 0.38430243730545044, + "rewards/tag_count_reward/mean": 0.9130859375, + "rewards/tag_count_reward/std": 0.19663172960281372, + "step": 1596 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2014.0, + "completions/mean_length": 864.05078125, + "completions/mean_terminated_length": 800.7119140625, + "completions/min_length": 4.0, + "completions/min_terminated_length": 4.0, + "epoch": 0.5451907484851071, + "grad_norm": 2.446843385696411, + "kl": 5.05859375, + "learning_rate": 5.577731788253762e-07, + "loss": 0.3345, + "num_tokens": 891413807.0, + "reward": 1.8291015625, + "reward_std": 0.5291173458099365, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.9189453125, + "rewards/tag_count_reward/std": 0.20217134058475494, + "step": 1597 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 829.580078125, + "completions/mean_terminated_length": 782.6226806640625, + "completions/min_length": 174.0, + "completions/min_terminated_length": 174.0, + "epoch": 0.5455321327984979, + "grad_norm": 2.638470411300659, + "kl": 6.1953125, + "learning_rate": 5.572371454628981e-07, + "loss": 0.4269, + "num_tokens": 891920488.0, + "reward": 1.7822265625, + "reward_std": 0.5422258377075195, + "rewards/accuracy_reward/mean": 0.0234375, + "rewards/accuracy_reward/std": 0.15143637359142303, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.9130859375, + "rewards/tag_count_reward/std": 0.20215243101119995, + "step": 1598 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2007.0, + "completions/mean_length": 870.962890625, + "completions/mean_terminated_length": 792.4937744140625, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "epoch": 0.5458735171118887, + "grad_norm": 1.2676059007644653, + "kl": 7.953125, + "learning_rate": 5.567011018286159e-07, + "loss": 0.486, + "num_tokens": 892442341.0, + "reward": 1.80859375, + "reward_std": 0.6159893274307251, + "rewards/accuracy_reward/mean": 0.111328125, + "rewards/accuracy_reward/std": 0.31484565138816833, + "rewards/format_reward/mean": 0.802734375, + "rewards/format_reward/std": 0.3983237147331238, + "rewards/tag_count_reward/mean": 0.89453125, + "rewards/tag_count_reward/std": 0.22471851110458374, + "step": 1599 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1970.0, + "completions/mean_length": 940.267578125, + "completions/mean_terminated_length": 876.183837890625, + "completions/min_length": 41.0, + "completions/min_terminated_length": 41.0, + "epoch": 0.5462149014252795, + "grad_norm": 1.526657223701477, + "kl": 7.765625, + "learning_rate": 5.56165048683345e-07, + "loss": 0.4601, + "num_tokens": 892994782.0, + "reward": 1.76025390625, + "reward_std": 0.6191115379333496, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.802734375, + "rewards/format_reward/std": 0.3983237147331238, + "rewards/tag_count_reward/mean": 0.89306640625, + "rewards/tag_count_reward/std": 0.22592659294605255, + "step": 1600 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1981.0, + "completions/mean_length": 893.10546875, + "completions/mean_terminated_length": 831.3209838867188, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "epoch": 0.5465562857386703, + "grad_norm": 2.4214370250701904, + "kl": 7.84375, + "learning_rate": 5.556289867879155e-07, + "loss": 0.4828, + "num_tokens": 893529188.0, + "reward": 1.77783203125, + "reward_std": 0.5602582693099976, + "rewards/accuracy_reward/mean": 0.04233871027827263, + "rewards/accuracy_reward/std": 0.2015640139579773, + "rewards/format_reward/mean": 0.826171875, + "rewards/format_reward/std": 0.3793322443962097, + "rewards/tag_count_reward/mean": 0.91064453125, + "rewards/tag_count_reward/std": 0.2166028767824173, + "step": 1601 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2031.0, + "completions/mean_length": 900.703125, + "completions/mean_terminated_length": 836.8330078125, + "completions/min_length": 99.0, + "completions/min_terminated_length": 99.0, + "epoch": 0.5468976700520611, + "grad_norm": 1.773688554763794, + "kl": 7.90625, + "learning_rate": 5.550929169031685e-07, + "loss": 0.4677, + "num_tokens": 894065436.0, + "reward": 1.76171875, + "reward_std": 0.6085403561592102, + "rewards/accuracy_reward/mean": 0.052419353276491165, + "rewards/accuracy_reward/std": 0.22309619188308716, + "rewards/format_reward/mean": 0.80859375, + "rewards/format_reward/std": 0.3937928080558777, + "rewards/tag_count_reward/mean": 0.90234375, + "rewards/tag_count_reward/std": 0.2211524099111557, + "step": 1602 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2018.0, + "completions/mean_length": 908.662109375, + "completions/mean_terminated_length": 837.7490234375, + "completions/min_length": 154.0, + "completions/min_terminated_length": 154.0, + "epoch": 0.547239054365452, + "grad_norm": 1.2073875665664673, + "kl": 7.5625, + "learning_rate": 5.545568397899575e-07, + "loss": 0.4764, + "num_tokens": 894609439.0, + "reward": 1.75244140625, + "reward_std": 0.5862622261047363, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.810546875, + "rewards/format_reward/std": 0.3922513723373413, + "rewards/tag_count_reward/mean": 0.90283203125, + "rewards/tag_count_reward/std": 0.22164389491081238, + "step": 1603 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0703125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 881.427734375, + "completions/mean_terminated_length": 793.1996459960938, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "epoch": 0.5475804386788428, + "grad_norm": 1.180014729499817, + "kl": 8.765625, + "learning_rate": 5.540207562091459e-07, + "loss": 0.5739, + "num_tokens": 895143242.0, + "reward": 1.7490234375, + "reward_std": 0.611189603805542, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.791015625, + "rewards/format_reward/std": 0.40698084235191345, + "rewards/tag_count_reward/mean": 0.8916015625, + "rewards/tag_count_reward/std": 0.23401492834091187, + "step": 1604 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1982.0, + "completions/mean_length": 845.85546875, + "completions/mean_terminated_length": 807.0765991210938, + "completions/min_length": 229.0, + "completions/min_terminated_length": 229.0, + "epoch": 0.5479218229922335, + "grad_norm": 3.5608949661254883, + "kl": 5.453125, + "learning_rate": 5.534846669216062e-07, + "loss": 0.3658, + "num_tokens": 895648160.0, + "reward": 1.837890625, + "reward_std": 0.5191460847854614, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.927734375, + "rewards/tag_count_reward/std": 0.19130395352840424, + "step": 1605 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1976.0, + "completions/mean_length": 850.73828125, + "completions/mean_terminated_length": 814.6035766601562, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.5482632073056243, + "grad_norm": 1.6136350631713867, + "kl": 6.1640625, + "learning_rate": 5.529485726882193e-07, + "loss": 0.3918, + "num_tokens": 896154426.0, + "reward": 1.84619140625, + "reward_std": 0.5813133716583252, + "rewards/accuracy_reward/mean": 0.103515625, + "rewards/accuracy_reward/std": 0.30492907762527466, + "rewards/format_reward/mean": 0.826171875, + "rewards/format_reward/std": 0.3793322443962097, + "rewards/tag_count_reward/mean": 0.91650390625, + "rewards/tag_count_reward/std": 0.19964765012264252, + "step": 1606 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 849.88671875, + "completions/mean_terminated_length": 798.6436157226562, + "completions/min_length": 5.0, + "completions/min_terminated_length": 5.0, + "epoch": 0.5486045916190151, + "grad_norm": 0.8744932413101196, + "kl": 7.328125, + "learning_rate": 5.524124742698728e-07, + "loss": 0.4574, + "num_tokens": 896660128.0, + "reward": 1.77197265625, + "reward_std": 0.5420060157775879, + "rewards/accuracy_reward/mean": 0.03125, + "rewards/accuracy_reward/std": 0.17416280508041382, + "rewards/format_reward/mean": 0.83203125, + "rewards/format_reward/std": 0.374204158782959, + "rewards/tag_count_reward/mean": 0.90869140625, + "rewards/tag_count_reward/std": 0.20887276530265808, + "step": 1607 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1942.0, + "completions/mean_length": 869.92578125, + "completions/mean_terminated_length": 831.92333984375, + "completions/min_length": 5.0, + "completions/min_terminated_length": 5.0, + "epoch": 0.5489459759324059, + "grad_norm": 1.3320586681365967, + "kl": 5.09765625, + "learning_rate": 5.518763724274602e-07, + "loss": 0.305, + "num_tokens": 897178330.0, + "reward": 1.83740234375, + "reward_std": 0.5293657779693604, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.93115234375, + "rewards/tag_count_reward/std": 0.17906923592090607, + "step": 1608 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 916.75390625, + "completions/mean_terminated_length": 856.2345581054688, + "completions/min_length": 190.0, + "completions/min_terminated_length": 190.0, + "epoch": 0.5492873602457967, + "grad_norm": 0.7662155032157898, + "kl": 7.0859375, + "learning_rate": 5.513402679218801e-07, + "loss": 0.4624, + "num_tokens": 897725612.0, + "reward": 1.80908203125, + "reward_std": 0.5761713981628418, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.8203125, + "rewards/format_reward/std": 0.38430243730545044, + "rewards/tag_count_reward/mean": 0.91064453125, + "rewards/tag_count_reward/std": 0.21145978569984436, + "step": 1609 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 915.40625, + "completions/mean_terminated_length": 869.3658447265625, + "completions/min_length": 117.0, + "completions/min_terminated_length": 117.0, + "epoch": 0.5496287445591875, + "grad_norm": 1.8923580646514893, + "kl": 6.90625, + "learning_rate": 5.50804161514035e-07, + "loss": 0.4278, + "num_tokens": 898269036.0, + "reward": 1.818359375, + "reward_std": 0.5410223603248596, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.912109375, + "rewards/tag_count_reward/std": 0.20711380243301392, + "step": 1610 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1973.0, + "completions/mean_length": 884.74609375, + "completions/mean_terminated_length": 830.03271484375, + "completions/min_length": 133.0, + "completions/min_terminated_length": 133.0, + "epoch": 0.5499701288725783, + "grad_norm": 0.9083179831504822, + "kl": 6.171875, + "learning_rate": 5.502680539648296e-07, + "loss": 0.3791, + "num_tokens": 898798426.0, + "reward": 1.794921875, + "reward_std": 0.4967016875743866, + "rewards/accuracy_reward/mean": 0.025390625, + "rewards/accuracy_reward/std": 0.15746226906776428, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.923828125, + "rewards/tag_count_reward/std": 0.1948670893907547, + "step": 1611 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1998.0, + "completions/mean_length": 807.29296875, + "completions/mean_terminated_length": 751.5877075195312, + "completions/min_length": 171.0, + "completions/min_terminated_length": 171.0, + "epoch": 0.5503115131859692, + "grad_norm": 1.611915946006775, + "kl": 8.203125, + "learning_rate": 5.497319460351706e-07, + "loss": 0.521, + "num_tokens": 899285264.0, + "reward": 1.80908203125, + "reward_std": 0.6097793579101562, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.826171875, + "rewards/format_reward/std": 0.3793322443962097, + "rewards/tag_count_reward/mean": 0.90673828125, + "rewards/tag_count_reward/std": 0.21608169376850128, + "step": 1612 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 869.275390625, + "completions/mean_terminated_length": 816.35302734375, + "completions/min_length": 84.0, + "completions/min_terminated_length": 84.0, + "epoch": 0.5506528974993599, + "grad_norm": 1.7584127187728882, + "kl": 8.2578125, + "learning_rate": 5.491958384859652e-07, + "loss": 0.518, + "num_tokens": 899805005.0, + "reward": 1.80078125, + "reward_std": 0.5637913942337036, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.8359375, + "rewards/format_reward/std": 0.37069445848464966, + "rewards/tag_count_reward/mean": 0.9140625, + "rewards/tag_count_reward/std": 0.20910651981830597, + "step": 1613 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 876.923828125, + "completions/mean_terminated_length": 824.3448486328125, + "completions/min_length": 157.0, + "completions/min_terminated_length": 157.0, + "epoch": 0.5509942818127507, + "grad_norm": 2.293674945831299, + "kl": 7.4453125, + "learning_rate": 5.486597320781199e-07, + "loss": 0.4283, + "num_tokens": 900330390.0, + "reward": 1.80810546875, + "reward_std": 0.5948246717453003, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.8203125, + "rewards/format_reward/std": 0.38430243730545044, + "rewards/tag_count_reward/mean": 0.90771484375, + "rewards/tag_count_reward/std": 0.2054874747991562, + "step": 1614 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.072265625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1878.0, + "completions/mean_length": 945.734375, + "completions/mean_terminated_length": 859.8736572265625, + "completions/min_length": 180.0, + "completions/min_terminated_length": 180.0, + "epoch": 0.5513356661261415, + "grad_norm": 5.122795581817627, + "kl": 10.8125, + "learning_rate": 5.481236275725398e-07, + "loss": 0.6246, + "num_tokens": 900889022.0, + "reward": 1.654296875, + "reward_std": 0.6732208132743835, + "rewards/accuracy_reward/mean": 0.03629032149910927, + "rewards/accuracy_reward/std": 0.1872003972530365, + "rewards/format_reward/mean": 0.751953125, + "rewards/format_reward/std": 0.4323015511035919, + "rewards/tag_count_reward/mean": 0.8671875, + "rewards/tag_count_reward/std": 0.2496328055858612, + "step": 1615 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.07421875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1953.0, + "completions/mean_length": 970.21875, + "completions/mean_terminated_length": 883.8143310546875, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.5516770504395323, + "grad_norm": 6.714582920074463, + "kl": 9.9140625, + "learning_rate": 5.475875257301274e-07, + "loss": 0.5375, + "num_tokens": 901460302.0, + "reward": 1.6875, + "reward_std": 0.6485152244567871, + "rewards/accuracy_reward/mean": 0.06653226166963577, + "rewards/accuracy_reward/std": 0.24946178495883942, + "rewards/format_reward/mean": 0.75390625, + "rewards/format_reward/std": 0.4311550557613373, + "rewards/tag_count_reward/mean": 0.869140625, + "rewards/tag_count_reward/std": 0.2511516213417053, + "step": 1616 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1884.0, + "completions/mean_length": 868.125, + "completions/mean_terminated_length": 805.0040893554688, + "completions/min_length": 76.0, + "completions/min_terminated_length": 76.0, + "epoch": 0.5520184347529231, + "grad_norm": 1.7493685483932495, + "kl": 6.3671875, + "learning_rate": 5.470514273117807e-07, + "loss": 0.3642, + "num_tokens": 901984286.0, + "reward": 1.732421875, + "reward_std": 0.5994226932525635, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.791015625, + "rewards/format_reward/std": 0.40698084235191345, + "rewards/tag_count_reward/mean": 0.8984375, + "rewards/tag_count_reward/std": 0.21430610120296478, + "step": 1617 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1983.0, + "completions/mean_length": 786.9453125, + "completions/mean_terminated_length": 754.0922241210938, + "completions/min_length": 164.0, + "completions/min_terminated_length": 164.0, + "epoch": 0.5523598190663139, + "grad_norm": 3.0826311111450195, + "kl": 4.93359375, + "learning_rate": 5.465153330783939e-07, + "loss": 0.3272, + "num_tokens": 902468178.0, + "reward": 1.8212890625, + "reward_std": 0.49712318181991577, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.9248046875, + "rewards/tag_count_reward/std": 0.18952499330043793, + "step": 1618 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1954.0, + "completions/mean_length": 926.4609375, + "completions/mean_terminated_length": 851.6917114257812, + "completions/min_length": 5.0, + "completions/min_terminated_length": 5.0, + "epoch": 0.5527012033797047, + "grad_norm": 1.520570158958435, + "kl": 7.015625, + "learning_rate": 5.459792437908542e-07, + "loss": 0.4051, + "num_tokens": 903015198.0, + "reward": 1.72900390625, + "reward_std": 0.6388310194015503, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.763671875, + "rewards/format_reward/std": 0.42524150013923645, + "rewards/tag_count_reward/mean": 0.87939453125, + "rewards/tag_count_reward/std": 0.23560354113578796, + "step": 1619 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 866.72265625, + "completions/mean_terminated_length": 816.1996459960938, + "completions/min_length": 134.0, + "completions/min_terminated_length": 134.0, + "epoch": 0.5530425876930956, + "grad_norm": 1.5511140823364258, + "kl": 5.953125, + "learning_rate": 5.454431602100425e-07, + "loss": 0.4021, + "num_tokens": 903532928.0, + "reward": 1.75, + "reward_std": 0.5770016312599182, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.798828125, + "rewards/format_reward/std": 0.4012683033943176, + "rewards/tag_count_reward/mean": 0.896484375, + "rewards/tag_count_reward/std": 0.21508051455020905, + "step": 1620 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1964.0, + "completions/mean_length": 859.68359375, + "completions/mean_terminated_length": 831.1640625, + "completions/min_length": 178.0, + "completions/min_terminated_length": 178.0, + "epoch": 0.5533839720064863, + "grad_norm": 4.295713424682617, + "kl": 4.50390625, + "learning_rate": 5.449070830968316e-07, + "loss": 0.3205, + "num_tokens": 904046606.0, + "reward": 1.7939453125, + "reward_std": 0.5318611264228821, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.833984375, + "rewards/format_reward/std": 0.3724585771560669, + "rewards/tag_count_reward/mean": 0.9189453125, + "rewards/tag_count_reward/std": 0.1909715086221695, + "step": 1621 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1972.0, + "completions/mean_length": 860.353515625, + "completions/mean_terminated_length": 801.9446411132812, + "completions/min_length": 188.0, + "completions/min_terminated_length": 188.0, + "epoch": 0.5537253563198771, + "grad_norm": 1.6234852075576782, + "kl": 5.609375, + "learning_rate": 5.443710132120846e-07, + "loss": 0.3364, + "num_tokens": 904563795.0, + "reward": 1.79345703125, + "reward_std": 0.6144713163375854, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.802734375, + "rewards/format_reward/std": 0.3983237147331238, + "rewards/tag_count_reward/mean": 0.89892578125, + "rewards/tag_count_reward/std": 0.22099627554416656, + "step": 1622 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.060546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 910.89453125, + "completions/mean_terminated_length": 837.609130859375, + "completions/min_length": 24.0, + "completions/min_terminated_length": 24.0, + "epoch": 0.5540667406332679, + "grad_norm": 1.9639532566070557, + "kl": 6.671875, + "learning_rate": 5.438349513166549e-07, + "loss": 0.4267, + "num_tokens": 905106445.0, + "reward": 1.70458984375, + "reward_std": 0.66635662317276, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.24230584502220154, + "rewards/format_reward/mean": 0.767578125, + "rewards/format_reward/std": 0.42278963327407837, + "rewards/tag_count_reward/mean": 0.87646484375, + "rewards/tag_count_reward/std": 0.24380344152450562, + "step": 1623 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.060546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1860.0, + "completions/mean_length": 870.599609375, + "completions/mean_terminated_length": 794.71728515625, + "completions/min_length": 129.0, + "completions/min_terminated_length": 129.0, + "epoch": 0.5544081249466587, + "grad_norm": 1.7202931642532349, + "kl": 6.22265625, + "learning_rate": 5.432988981713842e-07, + "loss": 0.4141, + "num_tokens": 905624784.0, + "reward": 1.70654296875, + "reward_std": 0.6034641265869141, + "rewards/accuracy_reward/mean": 0.021484375, + "rewards/accuracy_reward/std": 0.14513419568538666, + "rewards/format_reward/mean": 0.787109375, + "rewards/format_reward/std": 0.409751296043396, + "rewards/tag_count_reward/mean": 0.89794921875, + "rewards/tag_count_reward/std": 0.21606400609016418, + "step": 1624 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.068359375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1966.0, + "completions/mean_length": 850.359375, + "completions/mean_terminated_length": 762.482177734375, + "completions/min_length": 151.0, + "completions/min_terminated_length": 151.0, + "epoch": 0.5547495092600495, + "grad_norm": 1.852662205696106, + "kl": 8.84375, + "learning_rate": 5.42762854537102e-07, + "loss": 0.5272, + "num_tokens": 906131304.0, + "reward": 1.78466796875, + "reward_std": 0.6673040390014648, + "rewards/accuracy_reward/mean": 0.119140625, + "rewards/accuracy_reward/std": 0.32427072525024414, + "rewards/format_reward/mean": 0.783203125, + "rewards/format_reward/std": 0.4124660789966583, + "rewards/tag_count_reward/mean": 0.88232421875, + "rewards/tag_count_reward/std": 0.23811282217502594, + "step": 1625 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.072265625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 900.119140625, + "completions/mean_terminated_length": 810.7052001953125, + "completions/min_length": 149.0, + "completions/min_terminated_length": 149.0, + "epoch": 0.5550908935734403, + "grad_norm": 4.000398635864258, + "kl": 9.5625, + "learning_rate": 5.422268211746239e-07, + "loss": 0.5557, + "num_tokens": 906667013.0, + "reward": 1.69775390625, + "reward_std": 0.6580989956855774, + "rewards/accuracy_reward/mean": 0.08266129344701767, + "rewards/accuracy_reward/std": 0.2756475806236267, + "rewards/format_reward/mean": 0.74609375, + "rewards/format_reward/std": 0.43567025661468506, + "rewards/tag_count_reward/mean": 0.87158203125, + "rewards/tag_count_reward/std": 0.24176867306232452, + "step": 1626 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.08984375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2010.0, + "completions/mean_length": 946.92578125, + "completions/mean_terminated_length": 838.2360229492188, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "epoch": 0.5554322778868311, + "grad_norm": 2.9227447509765625, + "kl": 9.234375, + "learning_rate": 5.416907988447514e-07, + "loss": 0.5736, + "num_tokens": 907233887.0, + "reward": 1.6416015625, + "reward_std": 0.6145917177200317, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.728515625, + "rewards/format_reward/std": 0.44516023993492126, + "rewards/tag_count_reward/mean": 0.8623046875, + "rewards/tag_count_reward/std": 0.25138550996780396, + "step": 1627 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1980.0, + "completions/mean_length": 846.671875, + "completions/mean_terminated_length": 795.291259765625, + "completions/min_length": 127.0, + "completions/min_terminated_length": 127.0, + "epoch": 0.555773662200222, + "grad_norm": 2.2842178344726562, + "kl": 7.78125, + "learning_rate": 5.411547883082701e-07, + "loss": 0.4494, + "num_tokens": 907753655.0, + "reward": 1.73486328125, + "reward_std": 0.641516387462616, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.76953125, + "rewards/format_reward/std": 0.42154473066329956, + "rewards/tag_count_reward/mean": 0.88720703125, + "rewards/tag_count_reward/std": 0.22953926026821136, + "step": 1628 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 849.826171875, + "completions/mean_terminated_length": 769.9479370117188, + "completions/min_length": 149.0, + "completions/min_terminated_length": 149.0, + "epoch": 0.5561150465136127, + "grad_norm": 2.6979892253875732, + "kl": 7.3671875, + "learning_rate": 5.406187903259491e-07, + "loss": 0.4321, + "num_tokens": 908262782.0, + "reward": 1.73681640625, + "reward_std": 0.6400578022003174, + "rewards/accuracy_reward/mean": 0.087890625, + "rewards/accuracy_reward/std": 0.2834126651287079, + "rewards/format_reward/mean": 0.7734375, + "rewards/format_reward/std": 0.4190165400505066, + "rewards/tag_count_reward/mean": 0.87548828125, + "rewards/tag_count_reward/std": 0.24630290269851685, + "step": 1629 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2010.0, + "completions/mean_length": 811.080078125, + "completions/mean_terminated_length": 758.17724609375, + "completions/min_length": 123.0, + "completions/min_terminated_length": 123.0, + "epoch": 0.5564564308270035, + "grad_norm": 4.622169017791748, + "kl": 5.68359375, + "learning_rate": 5.400828056585394e-07, + "loss": 0.4133, + "num_tokens": 908749751.0, + "reward": 1.83154296875, + "reward_std": 0.5589828491210938, + "rewards/accuracy_reward/mean": 0.07056451588869095, + "rewards/accuracy_reward/std": 0.25635457038879395, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.92529296875, + "rewards/tag_count_reward/std": 0.19761274755001068, + "step": 1630 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1956.0, + "completions/mean_length": 937.03125, + "completions/mean_terminated_length": 880.0000610351562, + "completions/min_length": 131.0, + "completions/min_terminated_length": 131.0, + "epoch": 0.5567978151403943, + "grad_norm": 3.613112211227417, + "kl": 6.390625, + "learning_rate": 5.395468350667732e-07, + "loss": 0.4359, + "num_tokens": 909317703.0, + "reward": 1.712890625, + "reward_std": 0.5855178833007812, + "rewards/accuracy_reward/mean": 0.029296875, + "rewards/accuracy_reward/std": 0.16880230605602264, + "rewards/format_reward/mean": 0.791015625, + "rewards/format_reward/std": 0.40698084235191345, + "rewards/tag_count_reward/mean": 0.892578125, + "rewards/tag_count_reward/std": 0.22542326152324677, + "step": 1631 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2016.0, + "completions/mean_length": 870.28515625, + "completions/mean_terminated_length": 812.3646850585938, + "completions/min_length": 141.0, + "completions/min_terminated_length": 141.0, + "epoch": 0.5571391994537851, + "grad_norm": 2.058533191680908, + "kl": 5.6484375, + "learning_rate": 5.390108793113628e-07, + "loss": 0.3515, + "num_tokens": 909843545.0, + "reward": 1.8017578125, + "reward_std": 0.5373117327690125, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.9169921875, + "rewards/tag_count_reward/std": 0.2055833637714386, + "step": 1632 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 847.859375, + "completions/mean_terminated_length": 796.529541015625, + "completions/min_length": 239.0, + "completions/min_terminated_length": 239.0, + "epoch": 0.5574805837671759, + "grad_norm": 4.070928573608398, + "kl": 4.33984375, + "learning_rate": 5.384749391529991e-07, + "loss": 0.2863, + "num_tokens": 910348753.0, + "reward": 1.833984375, + "reward_std": 0.48867836594581604, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.9296875, + "rewards/tag_count_reward/std": 0.1907537281513214, + "step": 1633 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1959.0, + "completions/mean_length": 812.2734375, + "completions/mean_terminated_length": 743.48046875, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "epoch": 0.5578219680805667, + "grad_norm": 1.5464138984680176, + "kl": 5.5234375, + "learning_rate": 5.379390153523515e-07, + "loss": 0.3656, + "num_tokens": 910844829.0, + "reward": 1.86328125, + "reward_std": 0.5772223472595215, + "rewards/accuracy_reward/mean": 0.111328125, + "rewards/accuracy_reward/std": 0.31484565138816833, + "rewards/format_reward/mean": 0.833984375, + "rewards/format_reward/std": 0.3724585771560669, + "rewards/tag_count_reward/mean": 0.91796875, + "rewards/tag_count_reward/std": 0.20715993642807007, + "step": 1634 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 811.873046875, + "completions/mean_terminated_length": 792.2520141601562, + "completions/min_length": 193.0, + "completions/min_terminated_length": 193.0, + "epoch": 0.5581633523939575, + "grad_norm": 2.051917552947998, + "kl": 3.96484375, + "learning_rate": 5.374031086700654e-07, + "loss": 0.2247, + "num_tokens": 911337500.0, + "reward": 1.8916015625, + "reward_std": 0.4719935953617096, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.8828125, + "rewards/format_reward/std": 0.32195815443992615, + "rewards/tag_count_reward/mean": 0.9443359375, + "rewards/tag_count_reward/std": 0.16794821619987488, + "step": 1635 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1928.0, + "completions/mean_length": 869.28515625, + "completions/mean_terminated_length": 828.8040771484375, + "completions/min_length": 208.0, + "completions/min_terminated_length": 208.0, + "epoch": 0.5585047367073483, + "grad_norm": 1.0205174684524536, + "kl": 4.921875, + "learning_rate": 5.368672198667627e-07, + "loss": 0.2909, + "num_tokens": 911863870.0, + "reward": 1.84375, + "reward_std": 0.5353987812995911, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.857421875, + "rewards/format_reward/std": 0.3499840497970581, + "rewards/tag_count_reward/mean": 0.921875, + "rewards/tag_count_reward/std": 0.2021169811487198, + "step": 1636 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2023.0, + "completions/mean_length": 855.365234375, + "completions/mean_terminated_length": 814.4060668945312, + "completions/min_length": 142.0, + "completions/min_terminated_length": 142.0, + "epoch": 0.558846121020739, + "grad_norm": 1.695725679397583, + "kl": 5.53125, + "learning_rate": 5.363313497030395e-07, + "loss": 0.3405, + "num_tokens": 912388569.0, + "reward": 1.8056640625, + "reward_std": 0.5462358593940735, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.83203125, + "rewards/format_reward/std": 0.374204158782959, + "rewards/tag_count_reward/mean": 0.9248046875, + "rewards/tag_count_reward/std": 0.1939898282289505, + "step": 1637 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1886.0, + "completions/mean_length": 819.974609375, + "completions/mean_terminated_length": 780.36083984375, + "completions/min_length": 159.0, + "completions/min_terminated_length": 159.0, + "epoch": 0.5591875053341299, + "grad_norm": 1.639160394668579, + "kl": 5.8046875, + "learning_rate": 5.357954989394651e-07, + "loss": 0.3393, + "num_tokens": 912889676.0, + "reward": 1.833984375, + "reward_std": 0.4923778772354126, + "rewards/accuracy_reward/mean": 0.04233871027827263, + "rewards/accuracy_reward/std": 0.2015640139579773, + "rewards/format_reward/mean": 0.857421875, + "rewards/format_reward/std": 0.3499840497970581, + "rewards/tag_count_reward/mean": 0.935546875, + "rewards/tag_count_reward/std": 0.18036192655563354, + "step": 1638 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 792.453125, + "completions/mean_terminated_length": 733.3987426757812, + "completions/min_length": 149.0, + "completions/min_terminated_length": 149.0, + "epoch": 0.5595288896475207, + "grad_norm": 4.835798740386963, + "kl": 8.6484375, + "learning_rate": 5.35259668336582e-07, + "loss": 0.4843, + "num_tokens": 913363940.0, + "reward": 1.8251953125, + "reward_std": 0.6062630414962769, + "rewards/accuracy_reward/mean": 0.1088709682226181, + "rewards/accuracy_reward/std": 0.31179171800613403, + "rewards/format_reward/mean": 0.810546875, + "rewards/format_reward/std": 0.3922513723373413, + "rewards/tag_count_reward/mean": 0.9091796875, + "rewards/tag_count_reward/std": 0.20761838555335999, + "step": 1639 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1971.0, + "completions/mean_length": 893.580078125, + "completions/mean_terminated_length": 839.2821655273438, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.5598702739609115, + "grad_norm": 2.5028481483459473, + "kl": 7.515625, + "learning_rate": 5.347238586549036e-07, + "loss": 0.4639, + "num_tokens": 913895389.0, + "reward": 1.7939453125, + "reward_std": 0.533664345741272, + "rewards/accuracy_reward/mean": 0.03125, + "rewards/accuracy_reward/std": 0.17416280508041382, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.9169921875, + "rewards/tag_count_reward/std": 0.20379072427749634, + "step": 1640 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 862.306640625, + "completions/mean_terminated_length": 824.0584716796875, + "completions/min_length": 133.0, + "completions/min_terminated_length": 133.0, + "epoch": 0.5602116582743023, + "grad_norm": 2.6302478313446045, + "kl": 7.7109375, + "learning_rate": 5.341880706549138e-07, + "loss": 0.4584, + "num_tokens": 914415514.0, + "reward": 1.85498046875, + "reward_std": 0.5276713967323303, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.857421875, + "rewards/format_reward/std": 0.3499840497970581, + "rewards/tag_count_reward/mean": 0.91943359375, + "rewards/tag_count_reward/std": 0.2014581561088562, + "step": 1641 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2018.0, + "completions/mean_length": 892.8046875, + "completions/mean_terminated_length": 815.7916870117188, + "completions/min_length": 167.0, + "completions/min_terminated_length": 167.0, + "epoch": 0.5605530425876931, + "grad_norm": 2.9713118076324463, + "kl": 10.234375, + "learning_rate": 5.336523050970657e-07, + "loss": 0.6381, + "num_tokens": 914950086.0, + "reward": 1.7939453125, + "reward_std": 0.6171619892120361, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.833984375, + "rewards/format_reward/std": 0.3724585771560669, + "rewards/tag_count_reward/mean": 0.8994140625, + "rewards/tag_count_reward/std": 0.23385153710842133, + "step": 1642 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1995.0, + "completions/mean_length": 819.419921875, + "completions/mean_terminated_length": 764.2591552734375, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "epoch": 0.5608944269010839, + "grad_norm": 1.6978282928466797, + "kl": 7.5625, + "learning_rate": 5.331165627417807e-07, + "loss": 0.4526, + "num_tokens": 915449837.0, + "reward": 1.82568359375, + "reward_std": 0.5862592458724976, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.91552734375, + "rewards/tag_count_reward/std": 0.21346116065979004, + "step": 1643 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 912.802734375, + "completions/mean_terminated_length": 847.130126953125, + "completions/min_length": 18.0, + "completions/min_terminated_length": 18.0, + "epoch": 0.5612358112144747, + "grad_norm": 2.8521158695220947, + "kl": 8.171875, + "learning_rate": 5.325808443494467e-07, + "loss": 0.471, + "num_tokens": 915999864.0, + "reward": 1.7509765625, + "reward_std": 0.5721356272697449, + "rewards/accuracy_reward/mean": 0.01953125, + "rewards/accuracy_reward/std": 0.1385180652141571, + "rewards/format_reward/mean": 0.82421875, + "rewards/format_reward/std": 0.3810062110424042, + "rewards/tag_count_reward/mean": 0.9072265625, + "rewards/tag_count_reward/std": 0.21826240420341492, + "step": 1644 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1944.0, + "completions/mean_length": 808.5234375, + "completions/mean_terminated_length": 758.13818359375, + "completions/min_length": 182.0, + "completions/min_terminated_length": 182.0, + "epoch": 0.5615771955278654, + "grad_norm": 1.2435574531555176, + "kl": 7.296875, + "learning_rate": 5.32045150680418e-07, + "loss": 0.4752, + "num_tokens": 916498244.0, + "reward": 1.880859375, + "reward_std": 0.6028008460998535, + "rewards/accuracy_reward/mean": 0.109375, + "rewards/accuracy_reward/std": 0.31241437792778015, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.91796875, + "rewards/tag_count_reward/std": 0.2011692225933075, + "step": 1645 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2033.0, + "completions/mean_length": 890.66796875, + "completions/mean_terminated_length": 833.7499389648438, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "epoch": 0.5619185798412563, + "grad_norm": 1.9105225801467896, + "kl": 7.1484375, + "learning_rate": 5.31509482495014e-07, + "loss": 0.4276, + "num_tokens": 917024906.0, + "reward": 1.7724609375, + "reward_std": 0.5939950942993164, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.814453125, + "rewards/format_reward/std": 0.38912075757980347, + "rewards/tag_count_reward/mean": 0.8974609375, + "rewards/tag_count_reward/std": 0.2211502492427826, + "step": 1646 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2001.0, + "completions/mean_length": 933.173828125, + "completions/mean_terminated_length": 873.5328979492188, + "completions/min_length": 155.0, + "completions/min_terminated_length": 155.0, + "epoch": 0.5622599641546471, + "grad_norm": 0.8320483565330505, + "kl": 5.8984375, + "learning_rate": 5.309738405535177e-07, + "loss": 0.3445, + "num_tokens": 917575187.0, + "reward": 1.80322265625, + "reward_std": 0.6222258806228638, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.8046875, + "rewards/format_reward/std": 0.3968288004398346, + "rewards/tag_count_reward/mean": 0.90087890625, + "rewards/tag_count_reward/std": 0.22407497465610504, + "step": 1647 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 890.234375, + "completions/mean_terminated_length": 843.1707153320312, + "completions/min_length": 113.0, + "completions/min_terminated_length": 113.0, + "epoch": 0.5626013484680379, + "grad_norm": 1.7899667024612427, + "kl": 4.73828125, + "learning_rate": 5.304382256161746e-07, + "loss": 0.3263, + "num_tokens": 918109163.0, + "reward": 1.84423828125, + "reward_std": 0.5855379700660706, + "rewards/accuracy_reward/mean": 0.1328125, + "rewards/accuracy_reward/std": 0.33970388770103455, + "rewards/format_reward/mean": 0.802734375, + "rewards/format_reward/std": 0.3983237147331238, + "rewards/tag_count_reward/mean": 0.90869140625, + "rewards/tag_count_reward/std": 0.2059241086244583, + "step": 1648 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1932.0, + "completions/mean_length": 914.560546875, + "completions/mean_terminated_length": 863.67138671875, + "completions/min_length": 247.0, + "completions/min_terminated_length": 247.0, + "epoch": 0.5629427327814287, + "grad_norm": 4.089898109436035, + "kl": 5.7265625, + "learning_rate": 5.299026384431929e-07, + "loss": 0.4065, + "num_tokens": 918650762.0, + "reward": 1.78173828125, + "reward_std": 0.5551834106445312, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.830078125, + "rewards/format_reward/std": 0.3759314715862274, + "rewards/tag_count_reward/mean": 0.91064453125, + "rewards/tag_count_reward/std": 0.2085477113723755, + "step": 1649 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1854.0, + "completions/mean_length": 801.73046875, + "completions/mean_terminated_length": 764.11669921875, + "completions/min_length": 52.0, + "completions/min_terminated_length": 52.0, + "epoch": 0.5632841170948195, + "grad_norm": 3.8572473526000977, + "kl": 4.4921875, + "learning_rate": 5.293670797947396e-07, + "loss": 0.3147, + "num_tokens": 919134496.0, + "reward": 1.84912109375, + "reward_std": 0.549421489238739, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.92333984375, + "rewards/tag_count_reward/std": 0.19436074793338776, + "step": 1650 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2016.0, + "completions/mean_length": 832.83203125, + "completions/mean_terminated_length": 801.1743774414062, + "completions/min_length": 192.0, + "completions/min_terminated_length": 192.0, + "epoch": 0.5636255014082103, + "grad_norm": 3.1188771724700928, + "kl": 3.615234375, + "learning_rate": 5.288315504309429e-07, + "loss": 0.2475, + "num_tokens": 919630778.0, + "reward": 1.88134765625, + "reward_std": 0.5292597413063049, + "rewards/accuracy_reward/mean": 0.087890625, + "rewards/accuracy_reward/std": 0.2834126651287079, + "rewards/format_reward/mean": 0.859375, + "rewards/format_reward/std": 0.3479743003845215, + "rewards/tag_count_reward/mean": 0.93408203125, + "rewards/tag_count_reward/std": 0.17743425071239471, + "step": 1651 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2011.0, + "completions/mean_length": 829.96875, + "completions/mean_terminated_length": 803.2255249023438, + "completions/min_length": 210.0, + "completions/min_terminated_length": 210.0, + "epoch": 0.5639668857216011, + "grad_norm": 2.7055652141571045, + "kl": 4.1328125, + "learning_rate": 5.282960511118882e-07, + "loss": 0.2534, + "num_tokens": 920136122.0, + "reward": 1.81787109375, + "reward_std": 0.5305625200271606, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.841796875, + "rewards/format_reward/std": 0.36528825759887695, + "rewards/tag_count_reward/mean": 0.92138671875, + "rewards/tag_count_reward/std": 0.1960885375738144, + "step": 1652 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1969.0, + "completions/mean_length": 832.1015625, + "completions/mean_terminated_length": 812.8016357421875, + "completions/min_length": 250.0, + "completions/min_terminated_length": 250.0, + "epoch": 0.5643082700349918, + "grad_norm": 3.7091875076293945, + "kl": 4.015625, + "learning_rate": 5.277605825976195e-07, + "loss": 0.2954, + "num_tokens": 920638990.0, + "reward": 1.85009765625, + "reward_std": 0.4466114640235901, + "rewards/accuracy_reward/mean": 0.037109375, + "rewards/accuracy_reward/std": 0.18921469151973724, + "rewards/format_reward/mean": 0.873046875, + "rewards/format_reward/std": 0.33324605226516724, + "rewards/tag_count_reward/mean": 0.93994140625, + "rewards/tag_count_reward/std": 0.1667913943529129, + "step": 1653 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1983.0, + "completions/mean_length": 823.515625, + "completions/mean_terminated_length": 776.3245239257812, + "completions/min_length": 153.0, + "completions/min_terminated_length": 153.0, + "epoch": 0.5646496543483827, + "grad_norm": 2.001434803009033, + "kl": 6.2421875, + "learning_rate": 5.272251456481363e-07, + "loss": 0.4146, + "num_tokens": 921139206.0, + "reward": 1.78466796875, + "reward_std": 0.5764614343643188, + "rewards/accuracy_reward/mean": 0.05443548411130905, + "rewards/accuracy_reward/std": 0.2271040678024292, + "rewards/format_reward/mean": 0.81640625, + "rewards/format_reward/std": 0.3875311613082886, + "rewards/tag_count_reward/mean": 0.91552734375, + "rewards/tag_count_reward/std": 0.19614213705062866, + "step": 1654 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2033.0, + "completions/mean_length": 879.6640625, + "completions/mean_terminated_length": 837.0931396484375, + "completions/min_length": 135.0, + "completions/min_terminated_length": 135.0, + "epoch": 0.5649910386617735, + "grad_norm": 2.615372896194458, + "kl": 6.3359375, + "learning_rate": 5.266897410233934e-07, + "loss": 0.3738, + "num_tokens": 921662618.0, + "reward": 1.79833984375, + "reward_std": 0.5633862018585205, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.8359375, + "rewards/format_reward/std": 0.37069445848464966, + "rewards/tag_count_reward/mean": 0.92333984375, + "rewards/tag_count_reward/std": 0.18665657937526703, + "step": 1655 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1918.0, + "completions/mean_length": 843.025390625, + "completions/mean_terminated_length": 788.9244384765625, + "completions/min_length": 172.0, + "completions/min_terminated_length": 172.0, + "epoch": 0.5653324229751643, + "grad_norm": 2.918985605239868, + "kl": 7.78125, + "learning_rate": 5.261543694832994e-07, + "loss": 0.4458, + "num_tokens": 922169671.0, + "reward": 1.7412109375, + "reward_std": 0.6137492656707764, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.796875, + "rewards/format_reward/std": 0.4027182459831238, + "rewards/tag_count_reward/mean": 0.9013671875, + "rewards/tag_count_reward/std": 0.21453560888767242, + "step": 1656 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1980.0, + "completions/mean_length": 812.248046875, + "completions/mean_terminated_length": 764.6226806640625, + "completions/min_length": 181.0, + "completions/min_terminated_length": 181.0, + "epoch": 0.5656738072885551, + "grad_norm": 3.5504391193389893, + "kl": 7.1171875, + "learning_rate": 5.256190317877164e-07, + "loss": 0.4138, + "num_tokens": 922658262.0, + "reward": 1.822265625, + "reward_std": 0.5267001390457153, + "rewards/accuracy_reward/mean": 0.0947580635547638, + "rewards/accuracy_reward/std": 0.29317617416381836, + "rewards/format_reward/mean": 0.8125, + "rewards/format_reward/std": 0.39069411158561707, + "rewards/tag_count_reward/mean": 0.91796875, + "rewards/tag_count_reward/std": 0.19499453902244568, + "step": 1657 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2013.0, + "completions/mean_length": 820.455078125, + "completions/mean_terminated_length": 767.9531860351562, + "completions/min_length": 98.0, + "completions/min_terminated_length": 98.0, + "epoch": 0.5660151916019459, + "grad_norm": 3.745274066925049, + "kl": 8.2265625, + "learning_rate": 5.250837286964585e-07, + "loss": 0.4714, + "num_tokens": 923160175.0, + "reward": 1.791015625, + "reward_std": 0.6207277178764343, + "rewards/accuracy_reward/mean": 0.0786290317773819, + "rewards/accuracy_reward/std": 0.26943066716194153, + "rewards/format_reward/mean": 0.80859375, + "rewards/format_reward/std": 0.3937928080558777, + "rewards/tag_count_reward/mean": 0.90625, + "rewards/tag_count_reward/std": 0.21215508878231049, + "step": 1658 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2030.0, + "completions/mean_length": 897.849609375, + "completions/mean_terminated_length": 848.6578979492188, + "completions/min_length": 19.0, + "completions/min_terminated_length": 19.0, + "epoch": 0.5663565759153367, + "grad_norm": 5.2440876960754395, + "kl": 9.34375, + "learning_rate": 5.245484609692906e-07, + "loss": 0.5069, + "num_tokens": 923700306.0, + "reward": 1.693359375, + "reward_std": 0.5803141593933105, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.767578125, + "rewards/format_reward/std": 0.42278963327407837, + "rewards/tag_count_reward/mean": 0.88671875, + "rewards/tag_count_reward/std": 0.22580444812774658, + "step": 1659 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.064453125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2030.0, + "completions/mean_length": 882.724609375, + "completions/mean_terminated_length": 802.4447021484375, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "epoch": 0.5666979602287275, + "grad_norm": 2.603729009628296, + "kl": 8.8125, + "learning_rate": 5.240132293659268e-07, + "loss": 0.5226, + "num_tokens": 924225749.0, + "reward": 1.69091796875, + "reward_std": 0.5992175936698914, + "rewards/accuracy_reward/mean": 0.01953125, + "rewards/accuracy_reward/std": 0.1385180652141571, + "rewards/format_reward/mean": 0.779296875, + "rewards/format_reward/std": 0.4151262938976288, + "rewards/tag_count_reward/mean": 0.89208984375, + "rewards/tag_count_reward/std": 0.22491775453090668, + "step": 1660 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2012.0, + "completions/mean_length": 824.6171875, + "completions/mean_terminated_length": 780.04052734375, + "completions/min_length": 155.0, + "completions/min_terminated_length": 155.0, + "epoch": 0.5670393445421182, + "grad_norm": 1.838841199874878, + "kl": 7.6953125, + "learning_rate": 5.234780346460314e-07, + "loss": 0.4604, + "num_tokens": 924729873.0, + "reward": 1.81298828125, + "reward_std": 0.6124602556228638, + "rewards/accuracy_reward/mean": 0.103515625, + "rewards/accuracy_reward/std": 0.30492907762527466, + "rewards/format_reward/mean": 0.80078125, + "rewards/format_reward/std": 0.39980348944664, + "rewards/tag_count_reward/mean": 0.90869140625, + "rewards/tag_count_reward/std": 0.20232902467250824, + "step": 1661 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1981.0, + "completions/mean_length": 824.201171875, + "completions/mean_terminated_length": 784.7237548828125, + "completions/min_length": 151.0, + "completions/min_terminated_length": 151.0, + "epoch": 0.567380728855509, + "grad_norm": 0.8633550405502319, + "kl": 5.0078125, + "learning_rate": 5.229428775692146e-07, + "loss": 0.2933, + "num_tokens": 925226664.0, + "reward": 1.8515625, + "reward_std": 0.5280296802520752, + "rewards/accuracy_reward/mean": 0.087890625, + "rewards/accuracy_reward/std": 0.2834126651287079, + "rewards/format_reward/mean": 0.8359375, + "rewards/format_reward/std": 0.37069445848464966, + "rewards/tag_count_reward/mean": 0.927734375, + "rewards/tag_count_reward/std": 0.18213331699371338, + "step": 1662 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1951.0, + "completions/mean_length": 889.419921875, + "completions/mean_terminated_length": 842.3231201171875, + "completions/min_length": 131.0, + "completions/min_terminated_length": 131.0, + "epoch": 0.5677221131688999, + "grad_norm": 4.076156139373779, + "kl": 4.75390625, + "learning_rate": 5.224077588950342e-07, + "loss": 0.3376, + "num_tokens": 925763871.0, + "reward": 1.857421875, + "reward_std": 0.5448045134544373, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.9296875, + "rewards/tag_count_reward/std": 0.18686699867248535, + "step": 1663 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1926.0, + "completions/mean_length": 776.384765625, + "completions/mean_terminated_length": 743.2565307617188, + "completions/min_length": 4.0, + "completions/min_terminated_length": 4.0, + "epoch": 0.5680634974822907, + "grad_norm": 1.9562805891036987, + "kl": 4.57421875, + "learning_rate": 5.218726793829936e-07, + "loss": 0.2657, + "num_tokens": 926248196.0, + "reward": 1.9189453125, + "reward_std": 0.5813266634941101, + "rewards/accuracy_reward/mean": 0.150390625, + "rewards/accuracy_reward/std": 0.35780346393585205, + "rewards/format_reward/mean": 0.841796875, + "rewards/format_reward/std": 0.36528825759887695, + "rewards/tag_count_reward/mean": 0.9267578125, + "rewards/tag_count_reward/std": 0.19661229848861694, + "step": 1664 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1957.0, + "completions/mean_length": 872.9765625, + "completions/mean_terminated_length": 825.2113647460938, + "completions/min_length": 160.0, + "completions/min_terminated_length": 160.0, + "epoch": 0.5684048817956815, + "grad_norm": 1.9900144338607788, + "kl": 5.4140625, + "learning_rate": 5.213376397925399e-07, + "loss": 0.3612, + "num_tokens": 926761240.0, + "reward": 1.8974609375, + "reward_std": 0.48910754919052124, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.9365234375, + "rewards/tag_count_reward/std": 0.1807086020708084, + "step": 1665 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.060546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2024.0, + "completions/mean_length": 897.74609375, + "completions/mean_terminated_length": 823.61328125, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "epoch": 0.5687462661090723, + "grad_norm": 2.6536097526550293, + "kl": 6.1171875, + "learning_rate": 5.208026408830641e-07, + "loss": 0.4124, + "num_tokens": 927295526.0, + "reward": 1.77392578125, + "reward_std": 0.5496322512626648, + "rewards/accuracy_reward/mean": 0.03427419438958168, + "rewards/accuracy_reward/std": 0.18211627006530762, + "rewards/format_reward/mean": 0.830078125, + "rewards/format_reward/std": 0.3759314715862274, + "rewards/tag_count_reward/mean": 0.91064453125, + "rewards/tag_count_reward/std": 0.21884989738464355, + "step": 1666 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 847.388671875, + "completions/mean_terminated_length": 796.0387573242188, + "completions/min_length": 185.0, + "completions/min_terminated_length": 185.0, + "epoch": 0.5690876504224631, + "grad_norm": 2.7732086181640625, + "kl": 6.015625, + "learning_rate": 5.202676834138993e-07, + "loss": 0.4163, + "num_tokens": 927803773.0, + "reward": 1.88623046875, + "reward_std": 0.5830451846122742, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.92724609375, + "rewards/tag_count_reward/std": 0.1927117109298706, + "step": 1667 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1985.0, + "completions/mean_length": 836.603515625, + "completions/mean_terminated_length": 810.0059814453125, + "completions/min_length": 163.0, + "completions/min_terminated_length": 163.0, + "epoch": 0.5694290347358539, + "grad_norm": 1.5123244524002075, + "kl": 5.6328125, + "learning_rate": 5.1973276814432e-07, + "loss": 0.3307, + "num_tokens": 928318402.0, + "reward": 1.814453125, + "reward_std": 0.5502277612686157, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.921875, + "rewards/tag_count_reward/std": 0.19968174397945404, + "step": 1668 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 823.88671875, + "completions/mean_terminated_length": 774.1259765625, + "completions/min_length": 196.0, + "completions/min_terminated_length": 196.0, + "epoch": 0.5697704190492446, + "grad_norm": 0.9966332912445068, + "kl": 6.6015625, + "learning_rate": 5.191978958335402e-07, + "loss": 0.3997, + "num_tokens": 928812200.0, + "reward": 1.82861328125, + "reward_std": 0.5804578065872192, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.830078125, + "rewards/format_reward/std": 0.3759314715862274, + "rewards/tag_count_reward/mean": 0.91650390625, + "rewards/tag_count_reward/std": 0.2056826800107956, + "step": 1669 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.060546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2013.0, + "completions/mean_length": 951.396484375, + "completions/mean_terminated_length": 880.721435546875, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "epoch": 0.5701118033626354, + "grad_norm": 2.6709368228912354, + "kl": 8.8671875, + "learning_rate": 5.186630672407133e-07, + "loss": 0.5241, + "num_tokens": 929378563.0, + "reward": 1.7861328125, + "reward_std": 0.6257163286209106, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.8046875, + "rewards/format_reward/std": 0.3968288004398346, + "rewards/tag_count_reward/mean": 0.9033203125, + "rewards/tag_count_reward/std": 0.21936286985874176, + "step": 1670 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.056640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2008.0, + "completions/mean_length": 848.033203125, + "completions/mean_terminated_length": 775.9855346679688, + "completions/min_length": 12.0, + "completions/min_terminated_length": 12.0, + "epoch": 0.5704531876760263, + "grad_norm": 3.107811689376831, + "kl": 10.359375, + "learning_rate": 5.181282831249311e-07, + "loss": 0.6312, + "num_tokens": 929890372.0, + "reward": 1.78662109375, + "reward_std": 0.6742834448814392, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.802734375, + "rewards/format_reward/std": 0.3983237147331238, + "rewards/tag_count_reward/mean": 0.89013671875, + "rewards/tag_count_reward/std": 0.2382572442293167, + "step": 1671 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 836.806640625, + "completions/mean_terminated_length": 795.2101440429688, + "completions/min_length": 159.0, + "completions/min_terminated_length": 159.0, + "epoch": 0.5707945719894171, + "grad_norm": 2.627119302749634, + "kl": 7.640625, + "learning_rate": 5.175935442452213e-07, + "loss": 0.462, + "num_tokens": 930398993.0, + "reward": 1.77978515625, + "reward_std": 0.5478862524032593, + "rewards/accuracy_reward/mean": 0.038306452333927155, + "rewards/accuracy_reward/std": 0.19212883710861206, + "rewards/format_reward/mean": 0.82421875, + "rewards/format_reward/std": 0.3810062110424042, + "rewards/tag_count_reward/mean": 0.91845703125, + "rewards/tag_count_reward/std": 0.19551268219947815, + "step": 1672 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1884.0, + "completions/mean_length": 862.94140625, + "completions/mean_terminated_length": 819.7611694335938, + "completions/min_length": 189.0, + "completions/min_terminated_length": 189.0, + "epoch": 0.5711359563028079, + "grad_norm": 2.356896162033081, + "kl": 6.875, + "learning_rate": 5.170588513605485e-07, + "loss": 0.3958, + "num_tokens": 930923731.0, + "reward": 1.8046875, + "reward_std": 0.5468517541885376, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.91015625, + "rewards/tag_count_reward/std": 0.21555092930793762, + "step": 1673 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1956.0, + "completions/mean_length": 857.18359375, + "completions/mean_terminated_length": 793.4773559570312, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "epoch": 0.5714773406161987, + "grad_norm": 1.1741122007369995, + "kl": 7.671875, + "learning_rate": 5.165242052298112e-07, + "loss": 0.4698, + "num_tokens": 931438017.0, + "reward": 1.7880859375, + "reward_std": 0.6110674142837524, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.8203125, + "rewards/format_reward/std": 0.38430243730545044, + "rewards/tag_count_reward/mean": 0.9072265625, + "rewards/tag_count_reward/std": 0.2143038660287857, + "step": 1674 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 843.484375, + "completions/mean_terminated_length": 786.8302612304688, + "completions/min_length": 8.0, + "completions/min_terminated_length": 8.0, + "epoch": 0.5718187249295895, + "grad_norm": 0.7875505685806274, + "kl": 6.90625, + "learning_rate": 5.159896066118417e-07, + "loss": 0.4293, + "num_tokens": 931949369.0, + "reward": 1.83056640625, + "reward_std": 0.5713907480239868, + "rewards/accuracy_reward/mean": 0.07083333283662796, + "rewards/accuracy_reward/std": 0.2568138837814331, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.91650390625, + "rewards/tag_count_reward/std": 0.20863474905490875, + "step": 1675 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 864.046875, + "completions/mean_terminated_length": 785.11669921875, + "completions/min_length": 154.0, + "completions/min_terminated_length": 154.0, + "epoch": 0.5721601092429803, + "grad_norm": 2.757953405380249, + "kl": 7.40625, + "learning_rate": 5.15455056265405e-07, + "loss": 0.5266, + "num_tokens": 932473153.0, + "reward": 1.81494140625, + "reward_std": 0.5837230682373047, + "rewards/accuracy_reward/mean": 0.06451612710952759, + "rewards/accuracy_reward/std": 0.2459181249141693, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.91455078125, + "rewards/tag_count_reward/std": 0.2078409045934677, + "step": 1676 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 829.92578125, + "completions/mean_terminated_length": 795.6826782226562, + "completions/min_length": 185.0, + "completions/min_terminated_length": 185.0, + "epoch": 0.572501493556371, + "grad_norm": 2.9485995769500732, + "kl": 4.1484375, + "learning_rate": 5.149205549491975e-07, + "loss": 0.2912, + "num_tokens": 932972811.0, + "reward": 1.93408203125, + "reward_std": 0.4794820547103882, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.89453125, + "rewards/format_reward/std": 0.3074568510055542, + "rewards/tag_count_reward/mean": 0.94189453125, + "rewards/tag_count_reward/std": 0.17322689294815063, + "step": 1677 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 866.876953125, + "completions/mean_terminated_length": 806.244384765625, + "completions/min_length": 197.0, + "completions/min_terminated_length": 197.0, + "epoch": 0.5728428778697618, + "grad_norm": 2.8530094623565674, + "kl": 6.765625, + "learning_rate": 5.143861034218462e-07, + "loss": 0.4756, + "num_tokens": 933490908.0, + "reward": 1.81005859375, + "reward_std": 0.5465816259384155, + "rewards/accuracy_reward/mean": 0.05645161122083664, + "rewards/accuracy_reward/std": 0.23102475702762604, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.91552734375, + "rewards/tag_count_reward/std": 0.20999513566493988, + "step": 1678 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2019.0, + "completions/mean_length": 844.345703125, + "completions/mean_terminated_length": 792.8656005859375, + "completions/min_length": 148.0, + "completions/min_terminated_length": 148.0, + "epoch": 0.5731842621831527, + "grad_norm": 2.1478755474090576, + "kl": 6.59375, + "learning_rate": 5.138517024419071e-07, + "loss": 0.4221, + "num_tokens": 934008301.0, + "reward": 1.80029296875, + "reward_std": 0.5182666778564453, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.833984375, + "rewards/format_reward/std": 0.3724585771560669, + "rewards/tag_count_reward/mean": 0.92138671875, + "rewards/tag_count_reward/std": 0.1960885375738144, + "step": 1679 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2019.0, + "completions/mean_length": 815.40234375, + "completions/mean_terminated_length": 775.64111328125, + "completions/min_length": 189.0, + "completions/min_terminated_length": 189.0, + "epoch": 0.5735256464965435, + "grad_norm": 2.8920838832855225, + "kl": 5.71875, + "learning_rate": 5.13317352767865e-07, + "loss": 0.4276, + "num_tokens": 934500219.0, + "reward": 1.87109375, + "reward_std": 0.49277210235595703, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.876953125, + "rewards/format_reward/std": 0.32881227135658264, + "rewards/tag_count_reward/mean": 0.939453125, + "rewards/tag_count_reward/std": 0.17274148762226105, + "step": 1680 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2025.0, + "completions/mean_length": 855.181640625, + "completions/mean_terminated_length": 780.9398803710938, + "completions/min_length": 5.0, + "completions/min_terminated_length": 5.0, + "epoch": 0.5738670308099343, + "grad_norm": 1.3681129217147827, + "kl": 7.6953125, + "learning_rate": 5.127830551581311e-07, + "loss": 0.471, + "num_tokens": 935022392.0, + "reward": 1.79150390625, + "reward_std": 0.5700950622558594, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.830078125, + "rewards/format_reward/std": 0.3759314715862274, + "rewards/tag_count_reward/mean": 0.90478515625, + "rewards/tag_count_reward/std": 0.22194115817546844, + "step": 1681 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.056640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1932.0, + "completions/mean_length": 890.224609375, + "completions/mean_terminated_length": 820.7101440429688, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "epoch": 0.5742084151233251, + "grad_norm": 4.133235454559326, + "kl": 9.375, + "learning_rate": 5.122488103710435e-07, + "loss": 0.5516, + "num_tokens": 935552955.0, + "reward": 1.77099609375, + "reward_std": 0.5811473727226257, + "rewards/accuracy_reward/mean": 0.060483869165182114, + "rewards/accuracy_reward/std": 0.2386218160390854, + "rewards/format_reward/mean": 0.810546875, + "rewards/format_reward/std": 0.3922513723373413, + "rewards/tag_count_reward/mean": 0.90185546875, + "rewards/tag_count_reward/std": 0.22065876424312592, + "step": 1682 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1995.0, + "completions/mean_length": 836.12109375, + "completions/mean_terminated_length": 773.90966796875, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "epoch": 0.5745497994367159, + "grad_norm": 1.4778292179107666, + "kl": 6.5078125, + "learning_rate": 5.117146191648647e-07, + "loss": 0.4152, + "num_tokens": 936056713.0, + "reward": 1.798828125, + "reward_std": 0.493483304977417, + "rewards/accuracy_reward/mean": 0.033203125, + "rewards/accuracy_reward/std": 0.17934183776378632, + "rewards/format_reward/mean": 0.841796875, + "rewards/format_reward/std": 0.36528825759887695, + "rewards/tag_count_reward/mean": 0.923828125, + "rewards/tag_count_reward/std": 0.19736173748970032, + "step": 1683 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2022.0, + "completions/mean_length": 853.40234375, + "completions/mean_terminated_length": 799.767333984375, + "completions/min_length": 171.0, + "completions/min_terminated_length": 171.0, + "epoch": 0.5748911837501067, + "grad_norm": 3.5044214725494385, + "kl": 8.3828125, + "learning_rate": 5.111804822977814e-07, + "loss": 0.5136, + "num_tokens": 936565591.0, + "reward": 1.8291015625, + "reward_std": 0.5825339555740356, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.822265625, + "rewards/format_reward/std": 0.3826628625392914, + "rewards/tag_count_reward/mean": 0.9091796875, + "rewards/tag_count_reward/std": 0.21342815458774567, + "step": 1684 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 775.412109375, + "completions/mean_terminated_length": 739.6365356445312, + "completions/min_length": 176.0, + "completions/min_terminated_length": 176.0, + "epoch": 0.5752325680634974, + "grad_norm": 1.9510211944580078, + "kl": 5.9609375, + "learning_rate": 5.106464005279034e-07, + "loss": 0.3894, + "num_tokens": 937036314.0, + "reward": 1.90869140625, + "reward_std": 0.5322166681289673, + "rewards/accuracy_reward/mean": 0.119140625, + "rewards/accuracy_reward/std": 0.32427072525024414, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.92822265625, + "rewards/tag_count_reward/std": 0.18794220685958862, + "step": 1685 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2002.0, + "completions/mean_length": 768.40234375, + "completions/mean_terminated_length": 705.4712524414062, + "completions/min_length": 102.0, + "completions/min_terminated_length": 102.0, + "epoch": 0.5755739523768882, + "grad_norm": 1.8461062908172607, + "kl": 7.65625, + "learning_rate": 5.101123746132622e-07, + "loss": 0.492, + "num_tokens": 937505576.0, + "reward": 1.84912109375, + "reward_std": 0.5417767763137817, + "rewards/accuracy_reward/mean": 0.087890625, + "rewards/accuracy_reward/std": 0.2834126651287079, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.92138671875, + "rewards/tag_count_reward/std": 0.20101657509803772, + "step": 1686 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2012.0, + "completions/mean_length": 860.7890625, + "completions/mean_terminated_length": 792.107421875, + "completions/min_length": 214.0, + "completions/min_terminated_length": 214.0, + "epoch": 0.5759153366902791, + "grad_norm": 1.1507009267807007, + "kl": 7.0703125, + "learning_rate": 5.095784053118094e-07, + "loss": 0.4438, + "num_tokens": 938025740.0, + "reward": 1.78466796875, + "reward_std": 0.5180612206459045, + "rewards/accuracy_reward/mean": 0.03515625, + "rewards/accuracy_reward/std": 0.1843547374010086, + "rewards/format_reward/mean": 0.83203125, + "rewards/format_reward/std": 0.374204158782959, + "rewards/tag_count_reward/mean": 0.91748046875, + "rewards/tag_count_reward/std": 0.203689306974411, + "step": 1687 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1985.0, + "completions/mean_length": 847.638671875, + "completions/mean_terminated_length": 798.8434448242188, + "completions/min_length": 4.0, + "completions/min_terminated_length": 4.0, + "epoch": 0.5762567210036699, + "grad_norm": 2.3126566410064697, + "kl": 6.3125, + "learning_rate": 5.090444933814171e-07, + "loss": 0.4363, + "num_tokens": 938552275.0, + "reward": 1.7841796875, + "reward_std": 0.5397467613220215, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.822265625, + "rewards/format_reward/std": 0.3826628625392914, + "rewards/tag_count_reward/mean": 0.9208984375, + "rewards/tag_count_reward/std": 0.193694069981575, + "step": 1688 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2001.0, + "completions/mean_length": 813.15234375, + "completions/mean_terminated_length": 768.1578979492188, + "completions/min_length": 182.0, + "completions/min_terminated_length": 182.0, + "epoch": 0.5765981053170607, + "grad_norm": 1.8903048038482666, + "kl": 5.06640625, + "learning_rate": 5.085106395798756e-07, + "loss": 0.3525, + "num_tokens": 939049937.0, + "reward": 1.93359375, + "reward_std": 0.4996389150619507, + "rewards/accuracy_reward/mean": 0.115234375, + "rewards/accuracy_reward/std": 0.3196168541908264, + "rewards/format_reward/mean": 0.87890625, + "rewards/format_reward/std": 0.3265552520751953, + "rewards/tag_count_reward/mean": 0.939453125, + "rewards/tag_count_reward/std": 0.17060412466526031, + "step": 1689 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.064453125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1924.0, + "completions/mean_length": 900.833984375, + "completions/mean_terminated_length": 821.8016967773438, + "completions/min_length": 187.0, + "completions/min_terminated_length": 187.0, + "epoch": 0.5769394896304515, + "grad_norm": 1.2121962308883667, + "kl": 7.21875, + "learning_rate": 5.079768446648926e-07, + "loss": 0.4765, + "num_tokens": 939583628.0, + "reward": 1.75390625, + "reward_std": 0.6164195537567139, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.7890625, + "rewards/format_reward/std": 0.4083731174468994, + "rewards/tag_count_reward/mean": 0.900390625, + "rewards/tag_count_reward/std": 0.21579019725322723, + "step": 1690 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1984.0, + "completions/mean_length": 801.580078125, + "completions/mean_terminated_length": 742.9550170898438, + "completions/min_length": 156.0, + "completions/min_terminated_length": 156.0, + "epoch": 0.5772808739438423, + "grad_norm": 1.0426567792892456, + "kl": 5.421875, + "learning_rate": 5.074431093940927e-07, + "loss": 0.3279, + "num_tokens": 940070069.0, + "reward": 1.841796875, + "reward_std": 0.5356305837631226, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.9140625, + "rewards/tag_count_reward/std": 0.20497123897075653, + "step": 1691 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1959.0, + "completions/mean_length": 825.408203125, + "completions/mean_terminated_length": 773.1181640625, + "completions/min_length": 160.0, + "completions/min_terminated_length": 160.0, + "epoch": 0.5776222582572331, + "grad_norm": 1.4213882684707642, + "kl": 5.96875, + "learning_rate": 5.069094345250152e-07, + "loss": 0.3999, + "num_tokens": 940563334.0, + "reward": 1.83447265625, + "reward_std": 0.5573749542236328, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.91650390625, + "rewards/tag_count_reward/std": 0.20686857402324677, + "step": 1692 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1952.0, + "completions/mean_length": 771.3515625, + "completions/mean_terminated_length": 727.507080078125, + "completions/min_length": 114.0, + "completions/min_terminated_length": 114.0, + "epoch": 0.5779636425706238, + "grad_norm": 2.150052547454834, + "kl": 5.109375, + "learning_rate": 5.063758208151139e-07, + "loss": 0.3768, + "num_tokens": 941031626.0, + "reward": 1.94140625, + "reward_std": 0.5016964673995972, + "rewards/accuracy_reward/mean": 0.126953125, + "rewards/accuracy_reward/std": 0.33324605226516724, + "rewards/format_reward/mean": 0.880859375, + "rewards/format_reward/std": 0.32427072525024414, + "rewards/tag_count_reward/mean": 0.93359375, + "rewards/tag_count_reward/std": 0.18501698970794678, + "step": 1693 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2019.0, + "completions/mean_length": 819.5234375, + "completions/mean_terminated_length": 743.062255859375, + "completions/min_length": 117.0, + "completions/min_terminated_length": 117.0, + "epoch": 0.5783050268840146, + "grad_norm": 1.2886264324188232, + "kl": 5.40625, + "learning_rate": 5.058422690217559e-07, + "loss": 0.3455, + "num_tokens": 941536326.0, + "reward": 1.86865234375, + "reward_std": 0.49906250834465027, + "rewards/accuracy_reward/mean": 0.07459677755832672, + "rewards/accuracy_reward/std": 0.263004869222641, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.93115234375, + "rewards/tag_count_reward/std": 0.18445254862308502, + "step": 1694 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 911.7421875, + "completions/mean_terminated_length": 848.4866333007812, + "completions/min_length": 152.0, + "completions/min_terminated_length": 152.0, + "epoch": 0.5786464111974055, + "grad_norm": 1.5921692848205566, + "kl": 7.015625, + "learning_rate": 5.053087799022207e-07, + "loss": 0.4269, + "num_tokens": 942081234.0, + "reward": 1.74951171875, + "reward_std": 0.5302519202232361, + "rewards/accuracy_reward/mean": 0.02734375, + "rewards/accuracy_reward/std": 0.16324250400066376, + "rewards/format_reward/mean": 0.81640625, + "rewards/format_reward/std": 0.3875311613082886, + "rewards/tag_count_reward/mean": 0.90576171875, + "rewards/tag_count_reward/std": 0.21847409009933472, + "step": 1695 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1804.0, + "completions/mean_length": 732.23828125, + "completions/mean_terminated_length": 700.6600341796875, + "completions/min_length": 115.0, + "completions/min_terminated_length": 115.0, + "epoch": 0.5789877955107963, + "grad_norm": 3.2324378490448, + "kl": 5.046875, + "learning_rate": 5.047753542136981e-07, + "loss": 0.3475, + "num_tokens": 942526796.0, + "reward": 1.8984375, + "reward_std": 0.5194200873374939, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.875, + "rewards/format_reward/std": 0.3310423493385315, + "rewards/tag_count_reward/mean": 0.943359375, + "rewards/tag_count_reward/std": 0.16834884881973267, + "step": 1696 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2031.0, + "completions/mean_length": 866.65625, + "completions/mean_terminated_length": 813.6162719726562, + "completions/min_length": 150.0, + "completions/min_terminated_length": 150.0, + "epoch": 0.5793291798241871, + "grad_norm": 2.79946231842041, + "kl": 5.9140625, + "learning_rate": 5.042419927132886e-07, + "loss": 0.3651, + "num_tokens": 943055228.0, + "reward": 1.83203125, + "reward_std": 0.5605237483978271, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.9140625, + "rewards/tag_count_reward/std": 0.2096906155347824, + "step": 1697 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1992.0, + "completions/mean_length": 821.318359375, + "completions/mean_terminated_length": 791.8780517578125, + "completions/min_length": 110.0, + "completions/min_terminated_length": 110.0, + "epoch": 0.5796705641375779, + "grad_norm": 1.3076624870300293, + "kl": 6.078125, + "learning_rate": 5.037086961580012e-07, + "loss": 0.3656, + "num_tokens": 943555407.0, + "reward": 1.85546875, + "reward_std": 0.5533447861671448, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.9296875, + "rewards/tag_count_reward/std": 0.18752038478851318, + "step": 1698 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2006.0, + "completions/mean_length": 859.642578125, + "completions/mean_terminated_length": 793.4866333007812, + "completions/min_length": 178.0, + "completions/min_terminated_length": 178.0, + "epoch": 0.5800119484509687, + "grad_norm": 2.272294521331787, + "kl": 6.4921875, + "learning_rate": 5.031754653047528e-07, + "loss": 0.375, + "num_tokens": 944063832.0, + "reward": 1.8642578125, + "reward_std": 0.5668913125991821, + "rewards/accuracy_reward/mean": 0.107421875, + "rewards/accuracy_reward/std": 0.30995169281959534, + "rewards/format_reward/mean": 0.841796875, + "rewards/format_reward/std": 0.36528825759887695, + "rewards/tag_count_reward/mean": 0.9150390625, + "rewards/tag_count_reward/std": 0.2129799723625183, + "step": 1699 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1969.0, + "completions/mean_length": 864.37109375, + "completions/mean_terminated_length": 808.6993408203125, + "completions/min_length": 200.0, + "completions/min_terminated_length": 200.0, + "epoch": 0.5803533327643595, + "grad_norm": 1.033617377281189, + "kl": 6.515625, + "learning_rate": 5.02642300910367e-07, + "loss": 0.3874, + "num_tokens": 944594998.0, + "reward": 1.81494140625, + "reward_std": 0.5386810898780823, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.830078125, + "rewards/format_reward/std": 0.3759314715862274, + "rewards/tag_count_reward/mean": 0.92431640625, + "rewards/tag_count_reward/std": 0.1984763890504837, + "step": 1700 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2006.0, + "completions/mean_length": 885.51171875, + "completions/mean_terminated_length": 848.0120849609375, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "epoch": 0.5806947170777502, + "grad_norm": 1.0708470344543457, + "kl": 5.8671875, + "learning_rate": 5.021092037315733e-07, + "loss": 0.349, + "num_tokens": 945124076.0, + "reward": 1.80029296875, + "reward_std": 0.5660788416862488, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.818359375, + "rewards/format_reward/std": 0.38592514395713806, + "rewards/tag_count_reward/mean": 0.91943359375, + "rewards/tag_count_reward/std": 0.19340462982654572, + "step": 1701 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2017.0, + "completions/mean_length": 881.14453125, + "completions/mean_terminated_length": 813.6404418945312, + "completions/min_length": 112.0, + "completions/min_terminated_length": 112.0, + "epoch": 0.581036101391141, + "grad_norm": 1.2517772912979126, + "kl": 7.6171875, + "learning_rate": 5.015761745250055e-07, + "loss": 0.4515, + "num_tokens": 945645606.0, + "reward": 1.78955078125, + "reward_std": 0.6008893847465515, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.810546875, + "rewards/format_reward/std": 0.3922513723373413, + "rewards/tag_count_reward/mean": 0.91259765625, + "rewards/tag_count_reward/std": 0.20643390715122223, + "step": 1702 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1966.0, + "completions/mean_length": 856.904296875, + "completions/mean_terminated_length": 813.5040893554688, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "epoch": 0.5813774857045318, + "grad_norm": 2.9970943927764893, + "kl": 7.046875, + "learning_rate": 5.01043214047201e-07, + "loss": 0.392, + "num_tokens": 946160821.0, + "reward": 1.82958984375, + "reward_std": 0.5553572177886963, + "rewards/accuracy_reward/mean": 0.087890625, + "rewards/accuracy_reward/std": 0.2834126651287079, + "rewards/format_reward/mean": 0.826171875, + "rewards/format_reward/std": 0.3793322443962097, + "rewards/tag_count_reward/mean": 0.91552734375, + "rewards/tag_count_reward/std": 0.20408765971660614, + "step": 1703 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 835.712890625, + "completions/mean_terminated_length": 796.6068115234375, + "completions/min_length": 168.0, + "completions/min_terminated_length": 168.0, + "epoch": 0.5817188700179227, + "grad_norm": 2.7404091358184814, + "kl": 7.390625, + "learning_rate": 5.005103230546e-07, + "loss": 0.4008, + "num_tokens": 946661634.0, + "reward": 1.7939453125, + "reward_std": 0.5862789154052734, + "rewards/accuracy_reward/mean": 0.06854838877916336, + "rewards/accuracy_reward/std": 0.25293970108032227, + "rewards/format_reward/mean": 0.818359375, + "rewards/format_reward/std": 0.38592514395713806, + "rewards/tag_count_reward/mean": 0.9091796875, + "rewards/tag_count_reward/std": 0.21054330468177795, + "step": 1704 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 795.357421875, + "completions/mean_terminated_length": 757.55126953125, + "completions/min_length": 126.0, + "completions/min_terminated_length": 126.0, + "epoch": 0.5820602543313135, + "grad_norm": 2.0220682621002197, + "kl": 6.984375, + "learning_rate": 4.999775023035438e-07, + "loss": 0.3819, + "num_tokens": 947144441.0, + "reward": 1.91552734375, + "reward_std": 0.6095578670501709, + "rewards/accuracy_reward/mean": 0.177734375, + "rewards/accuracy_reward/std": 0.3826628625392914, + "rewards/format_reward/mean": 0.8203125, + "rewards/format_reward/std": 0.38430243730545044, + "rewards/tag_count_reward/mean": 0.91748046875, + "rewards/tag_count_reward/std": 0.19944173097610474, + "step": 1705 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1992.0, + "completions/mean_length": 829.986328125, + "completions/mean_terminated_length": 795.7449340820312, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "epoch": 0.5824016386447043, + "grad_norm": 2.52016019821167, + "kl": 5.5703125, + "learning_rate": 4.994447525502735e-07, + "loss": 0.3674, + "num_tokens": 947649106.0, + "reward": 1.85400390625, + "reward_std": 0.5111713409423828, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.93017578125, + "rewards/tag_count_reward/std": 0.18540844321250916, + "step": 1706 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2008.0, + "completions/mean_length": 858.845703125, + "completions/mean_terminated_length": 805.455078125, + "completions/min_length": 70.0, + "completions/min_terminated_length": 70.0, + "epoch": 0.5827430229580951, + "grad_norm": 2.769322633743286, + "kl": 6.6796875, + "learning_rate": 4.989120745509305e-07, + "loss": 0.466, + "num_tokens": 948168035.0, + "reward": 1.78564453125, + "reward_std": 0.6224857568740845, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.80859375, + "rewards/format_reward/std": 0.3937928080558777, + "rewards/tag_count_reward/mean": 0.89892578125, + "rewards/tag_count_reward/std": 0.22099627554416656, + "step": 1707 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1929.0, + "completions/mean_length": 824.849609375, + "completions/mean_terminated_length": 782.8424682617188, + "completions/min_length": 168.0, + "completions/min_terminated_length": 168.0, + "epoch": 0.5830844072714859, + "grad_norm": 3.5896201133728027, + "kl": 4.9921875, + "learning_rate": 4.983794690615535e-07, + "loss": 0.3309, + "num_tokens": 948672582.0, + "reward": 1.91357421875, + "reward_std": 0.5080065727233887, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.873046875, + "rewards/format_reward/std": 0.33324605226516724, + "rewards/tag_count_reward/mean": 0.93505859375, + "rewards/tag_count_reward/std": 0.19360215961933136, + "step": 1708 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 844.6953125, + "completions/mean_terminated_length": 803.3697509765625, + "completions/min_length": 112.0, + "completions/min_terminated_length": 112.0, + "epoch": 0.5834257915848767, + "grad_norm": 1.4315153360366821, + "kl": 5.24609375, + "learning_rate": 4.978469368380787e-07, + "loss": 0.2995, + "num_tokens": 949188458.0, + "reward": 1.79443359375, + "reward_std": 0.539413332939148, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.82421875, + "rewards/format_reward/std": 0.3810062110424042, + "rewards/tag_count_reward/mean": 0.91552734375, + "rewards/tag_count_reward/std": 0.20468609035015106, + "step": 1709 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1985.0, + "completions/mean_length": 808.55859375, + "completions/mean_terminated_length": 768.5765991210938, + "completions/min_length": 144.0, + "completions/min_terminated_length": 144.0, + "epoch": 0.5837671758982674, + "grad_norm": 2.2452232837677, + "kl": 4.375, + "learning_rate": 4.97314478636338e-07, + "loss": 0.2668, + "num_tokens": 949679192.0, + "reward": 1.9013671875, + "reward_std": 0.5118180513381958, + "rewards/accuracy_reward/mean": 0.099609375, + "rewards/accuracy_reward/std": 0.29977133870124817, + "rewards/format_reward/mean": 0.86328125, + "rewards/format_reward/std": 0.3438861668109894, + "rewards/tag_count_reward/mean": 0.9384765625, + "rewards/tag_count_reward/std": 0.1779806911945343, + "step": 1710 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2005.0, + "completions/mean_length": 793.44921875, + "completions/mean_terminated_length": 752.9797973632812, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "epoch": 0.5841085602116582, + "grad_norm": 3.081019163131714, + "kl": 4.6953125, + "learning_rate": 4.967820952120588e-07, + "loss": 0.3256, + "num_tokens": 950162430.0, + "reward": 1.85595703125, + "reward_std": 0.4917498230934143, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.873046875, + "rewards/format_reward/std": 0.33324605226516724, + "rewards/tag_count_reward/mean": 0.93603515625, + "rewards/tag_count_reward/std": 0.18154938519001007, + "step": 1711 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 854.888671875, + "completions/mean_terminated_length": 811.4149780273438, + "completions/min_length": 36.0, + "completions/min_terminated_length": 36.0, + "epoch": 0.5844499445250491, + "grad_norm": 2.012058973312378, + "kl": 5.9453125, + "learning_rate": 4.962497873208616e-07, + "loss": 0.4021, + "num_tokens": 950670741.0, + "reward": 1.845703125, + "reward_std": 0.5269229412078857, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.91796875, + "rewards/tag_count_reward/std": 0.2059757262468338, + "step": 1712 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2021.0, + "completions/mean_length": 841.134765625, + "completions/mean_terminated_length": 792.0751953125, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "epoch": 0.5847913288384399, + "grad_norm": 0.938133716583252, + "kl": 6.279296875, + "learning_rate": 4.957175557182601e-07, + "loss": 0.3664, + "num_tokens": 951185706.0, + "reward": 1.8212890625, + "reward_std": 0.5399194955825806, + "rewards/accuracy_reward/mean": 0.0786290317773819, + "rewards/accuracy_reward/std": 0.26943066716194153, + "rewards/format_reward/mean": 0.826171875, + "rewards/format_reward/std": 0.3793322443962097, + "rewards/tag_count_reward/mean": 0.9189453125, + "rewards/tag_count_reward/std": 0.1972721517086029, + "step": 1713 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1880.0, + "completions/mean_length": 802.322265625, + "completions/mean_terminated_length": 764.726318359375, + "completions/min_length": 201.0, + "completions/min_terminated_length": 201.0, + "epoch": 0.5851327131518307, + "grad_norm": 0.9956303238868713, + "kl": 6.3125, + "learning_rate": 4.9518540115966e-07, + "loss": 0.4025, + "num_tokens": 951675007.0, + "reward": 1.91455078125, + "reward_std": 0.5179139375686646, + "rewards/accuracy_reward/mean": 0.107421875, + "rewards/accuracy_reward/std": 0.30995169281959534, + "rewards/format_reward/mean": 0.86328125, + "rewards/format_reward/std": 0.3438861668109894, + "rewards/tag_count_reward/mean": 0.94384765625, + "rewards/tag_count_reward/std": 0.16814936697483063, + "step": 1714 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1898.0, + "completions/mean_length": 822.064453125, + "completions/mean_terminated_length": 790.1262817382812, + "completions/min_length": 178.0, + "completions/min_terminated_length": 178.0, + "epoch": 0.5854740974652215, + "grad_norm": 1.209726333618164, + "kl": 4.58203125, + "learning_rate": 4.946533244003572e-07, + "loss": 0.2774, + "num_tokens": 952168288.0, + "reward": 1.87060546875, + "reward_std": 0.4697340428829193, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.8828125, + "rewards/format_reward/std": 0.32195815443992615, + "rewards/tag_count_reward/mean": 0.94091796875, + "rewards/tag_count_reward/std": 0.17360158264636993, + "step": 1715 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2007.0, + "completions/mean_length": 874.486328125, + "completions/mean_terminated_length": 851.1095581054688, + "completions/min_length": 173.0, + "completions/min_terminated_length": 173.0, + "epoch": 0.5858154817786123, + "grad_norm": 0.7931808233261108, + "kl": 5.109375, + "learning_rate": 4.941213261955374e-07, + "loss": 0.3082, + "num_tokens": 952691609.0, + "reward": 1.89208984375, + "reward_std": 0.46906012296676636, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.890625, + "rewards/format_reward/std": 0.31241437792778015, + "rewards/tag_count_reward/mean": 0.94677734375, + "rewards/tag_count_reward/std": 0.16910135746002197, + "step": 1716 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 884.333984375, + "completions/mean_terminated_length": 834.564208984375, + "completions/min_length": 162.0, + "completions/min_terminated_length": 162.0, + "epoch": 0.5861568660920031, + "grad_norm": 3.3196003437042236, + "kl": 6.7265625, + "learning_rate": 4.935894073002749e-07, + "loss": 0.4011, + "num_tokens": 953222356.0, + "reward": 1.890625, + "reward_std": 0.5219039916992188, + "rewards/accuracy_reward/mean": 0.103515625, + "rewards/accuracy_reward/std": 0.30492907762527466, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.92578125, + "rewards/tag_count_reward/std": 0.19562077522277832, + "step": 1717 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2018.0, + "completions/mean_length": 835.525390625, + "completions/mean_terminated_length": 788.797119140625, + "completions/min_length": 180.0, + "completions/min_terminated_length": 180.0, + "epoch": 0.5864982504053938, + "grad_norm": 1.0607223510742188, + "kl": 6.09375, + "learning_rate": 4.930575684695309e-07, + "loss": 0.3787, + "num_tokens": 953724689.0, + "reward": 1.92138671875, + "reward_std": 0.4783501625061035, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.890625, + "rewards/format_reward/std": 0.31241437792778015, + "rewards/tag_count_reward/mean": 0.94091796875, + "rewards/tag_count_reward/std": 0.17846523225307465, + "step": 1718 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2021.0, + "completions/mean_length": 891.671875, + "completions/mean_terminated_length": 849.5385131835938, + "completions/min_length": 5.0, + "completions/min_terminated_length": 5.0, + "epoch": 0.5868396347187846, + "grad_norm": 1.6661996841430664, + "kl": 5.8203125, + "learning_rate": 4.925258104581534e-07, + "loss": 0.348, + "num_tokens": 954255721.0, + "reward": 1.87744140625, + "reward_std": 0.5170288681983948, + "rewards/accuracy_reward/mean": 0.109375, + "rewards/accuracy_reward/std": 0.31241437792778015, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.92431640625, + "rewards/tag_count_reward/std": 0.1941147744655609, + "step": 1719 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 801.8515625, + "completions/mean_terminated_length": 779.5546264648438, + "completions/min_length": 168.0, + "completions/min_terminated_length": 168.0, + "epoch": 0.5871810190321755, + "grad_norm": 1.2811676263809204, + "kl": 4.98046875, + "learning_rate": 4.91994134020876e-07, + "loss": 0.2976, + "num_tokens": 954739149.0, + "reward": 1.86865234375, + "reward_std": 0.46206235885620117, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.873046875, + "rewards/format_reward/std": 0.33324605226516724, + "rewards/tag_count_reward/mean": 0.94482421875, + "rewards/tag_count_reward/std": 0.1572064310312271, + "step": 1720 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1903.0, + "completions/mean_length": 826.0234375, + "completions/mean_terminated_length": 806.6270141601562, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "epoch": 0.5875224033455663, + "grad_norm": 1.8048064708709717, + "kl": 3.62890625, + "learning_rate": 4.914625399123159e-07, + "loss": 0.1979, + "num_tokens": 955234265.0, + "reward": 1.93994140625, + "reward_std": 0.48053503036499023, + "rewards/accuracy_reward/mean": 0.10080645233392715, + "rewards/accuracy_reward/std": 0.30137622356414795, + "rewards/format_reward/mean": 0.892578125, + "rewards/format_reward/std": 0.30995169281959534, + "rewards/tag_count_reward/mean": 0.94970703125, + "rewards/tag_count_reward/std": 0.1572915017604828, + "step": 1721 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2016.0, + "completions/mean_length": 867.78125, + "completions/mean_terminated_length": 794.3236694335938, + "completions/min_length": 150.0, + "completions/min_terminated_length": 150.0, + "epoch": 0.5878637876589571, + "grad_norm": 1.2541394233703613, + "kl": 6.0078125, + "learning_rate": 4.909310288869737e-07, + "loss": 0.3846, + "num_tokens": 955757769.0, + "reward": 1.8447265625, + "reward_std": 0.4987034499645233, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.9208984375, + "rewards/tag_count_reward/std": 0.20051981508731842, + "step": 1722 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2041.0, + "completions/mean_length": 888.212890625, + "completions/mean_terminated_length": 845.9534912109375, + "completions/min_length": 155.0, + "completions/min_terminated_length": 155.0, + "epoch": 0.5882051719723479, + "grad_norm": 0.8701521754264832, + "kl": 5.796875, + "learning_rate": 4.903996016992323e-07, + "loss": 0.3725, + "num_tokens": 956299974.0, + "reward": 1.83642578125, + "reward_std": 0.5082464218139648, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.92431640625, + "rewards/tag_count_reward/std": 0.18705546855926514, + "step": 1723 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1948.0, + "completions/mean_length": 873.1484375, + "completions/mean_terminated_length": 820.3999633789062, + "completions/min_length": 107.0, + "completions/min_terminated_length": 107.0, + "epoch": 0.5885465562857387, + "grad_norm": 1.3485100269317627, + "kl": 5.02734375, + "learning_rate": 4.898682591033551e-07, + "loss": 0.3417, + "num_tokens": 956830402.0, + "reward": 1.81201171875, + "reward_std": 0.4673128128051758, + "rewards/accuracy_reward/mean": 0.021484375, + "rewards/accuracy_reward/std": 0.14513419568538666, + "rewards/format_reward/mean": 0.86328125, + "rewards/format_reward/std": 0.3438861668109894, + "rewards/tag_count_reward/mean": 0.92724609375, + "rewards/tag_count_reward/std": 0.1927117109298706, + "step": 1724 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1985.0, + "completions/mean_length": 920.4375, + "completions/mean_terminated_length": 869.8121948242188, + "completions/min_length": 195.0, + "completions/min_terminated_length": 195.0, + "epoch": 0.5888879405991295, + "grad_norm": 2.1556503772735596, + "kl": 6.1875, + "learning_rate": 4.893370018534858e-07, + "loss": 0.3534, + "num_tokens": 957372210.0, + "reward": 1.75732421875, + "reward_std": 0.5546750426292419, + "rewards/accuracy_reward/mean": 0.02016128972172737, + "rewards/accuracy_reward/std": 0.14069372415542603, + "rewards/format_reward/mean": 0.828125, + "rewards/format_reward/std": 0.3776407241821289, + "rewards/tag_count_reward/mean": 0.90966796875, + "rewards/tag_count_reward/std": 0.20635519921779633, + "step": 1725 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2011.0, + "completions/mean_length": 928.28515625, + "completions/mean_terminated_length": 865.9505615234375, + "completions/min_length": 187.0, + "completions/min_terminated_length": 187.0, + "epoch": 0.5892293249125202, + "grad_norm": 1.8250174522399902, + "kl": 6.9453125, + "learning_rate": 4.888058307036468e-07, + "loss": 0.4175, + "num_tokens": 957924964.0, + "reward": 1.822265625, + "reward_std": 0.5986999273300171, + "rewards/accuracy_reward/mean": 0.1088709682226181, + "rewards/accuracy_reward/std": 0.31179171800613403, + "rewards/format_reward/mean": 0.8125, + "rewards/format_reward/std": 0.39069411158561707, + "rewards/tag_count_reward/mean": 0.904296875, + "rewards/tag_count_reward/std": 0.2118576318025589, + "step": 1726 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1777.0, + "completions/mean_length": 855.818359375, + "completions/mean_terminated_length": 797.1864013671875, + "completions/min_length": 70.0, + "completions/min_terminated_length": 70.0, + "epoch": 0.589570709225911, + "grad_norm": 2.7474205493927, + "kl": 6.890625, + "learning_rate": 4.882747464077388e-07, + "loss": 0.3981, + "num_tokens": 958442791.0, + "reward": 1.7666015625, + "reward_std": 0.5679394006729126, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.810546875, + "rewards/format_reward/std": 0.3922513723373413, + "rewards/tag_count_reward/mean": 0.9033203125, + "rewards/tag_count_reward/std": 0.21141289174556732, + "step": 1727 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1995.0, + "completions/mean_length": 798.84375, + "completions/mean_terminated_length": 753.3279418945312, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "epoch": 0.5899120935393019, + "grad_norm": 1.4245671033859253, + "kl": 5.1796875, + "learning_rate": 4.877437497195385e-07, + "loss": 0.3268, + "num_tokens": 958920935.0, + "reward": 1.80859375, + "reward_std": 0.5384195446968079, + "rewards/accuracy_reward/mean": 0.0463709682226181, + "rewards/accuracy_reward/std": 0.21049949526786804, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.91796875, + "rewards/tag_count_reward/std": 0.1931036412715912, + "step": 1728 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2029.0, + "completions/mean_length": 838.607421875, + "completions/mean_terminated_length": 784.30810546875, + "completions/min_length": 112.0, + "completions/min_terminated_length": 112.0, + "epoch": 0.5902534778526927, + "grad_norm": 2.010416030883789, + "kl": 4.71875, + "learning_rate": 4.872128413926989e-07, + "loss": 0.3042, + "num_tokens": 959439294.0, + "reward": 1.80419921875, + "reward_std": 0.5523759126663208, + "rewards/accuracy_reward/mean": 0.06653226166963577, + "rewards/accuracy_reward/std": 0.24946178495883942, + "rewards/format_reward/mean": 0.822265625, + "rewards/format_reward/std": 0.3826628625392914, + "rewards/tag_count_reward/mean": 0.91748046875, + "rewards/tag_count_reward/std": 0.18677429854869843, + "step": 1729 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1963.0, + "completions/mean_length": 782.833984375, + "completions/mean_terminated_length": 747.2670288085938, + "completions/min_length": 50.0, + "completions/min_terminated_length": 50.0, + "epoch": 0.5905948621660835, + "grad_norm": 2.0043551921844482, + "kl": 4.84765625, + "learning_rate": 4.866820221807467e-07, + "loss": 0.2975, + "num_tokens": 959918617.0, + "reward": 1.8447265625, + "reward_std": 0.5187762379646301, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.9267578125, + "rewards/tag_count_reward/std": 0.18374989926815033, + "step": 1730 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1909.0, + "completions/mean_length": 856.052734375, + "completions/mean_terminated_length": 817.602783203125, + "completions/min_length": 150.0, + "completions/min_terminated_length": 150.0, + "epoch": 0.5909362464794743, + "grad_norm": 1.4563902616500854, + "kl": 3.94140625, + "learning_rate": 4.861512928370831e-07, + "loss": 0.2507, + "num_tokens": 960432660.0, + "reward": 1.86376953125, + "reward_std": 0.49892061948776245, + "rewards/accuracy_reward/mean": 0.08669354766607285, + "rewards/accuracy_reward/std": 0.281669557094574, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.92822265625, + "rewards/tag_count_reward/std": 0.17514143884181976, + "step": 1731 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 857.337890625, + "completions/mean_terminated_length": 806.4134521484375, + "completions/min_length": 158.0, + "completions/min_terminated_length": 158.0, + "epoch": 0.5912776307928651, + "grad_norm": 2.9518094062805176, + "kl": 5.4375, + "learning_rate": 4.856206541149812e-07, + "loss": 0.3806, + "num_tokens": 960951377.0, + "reward": 1.7919921875, + "reward_std": 0.5659606456756592, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.818359375, + "rewards/format_reward/std": 0.38592514395713806, + "rewards/tag_count_reward/mean": 0.9091796875, + "rewards/tag_count_reward/std": 0.20643681287765503, + "step": 1732 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1987.0, + "completions/mean_length": 781.751953125, + "completions/mean_terminated_length": 746.1546020507812, + "completions/min_length": 191.0, + "completions/min_terminated_length": 191.0, + "epoch": 0.5916190151062559, + "grad_norm": 2.4586381912231445, + "kl": 4.390625, + "learning_rate": 4.850901067675855e-07, + "loss": 0.292, + "num_tokens": 961427042.0, + "reward": 1.86865234375, + "reward_std": 0.4505774974822998, + "rewards/accuracy_reward/mean": 0.038306452333927155, + "rewards/accuracy_reward/std": 0.19212883710861206, + "rewards/format_reward/mean": 0.88671875, + "rewards/format_reward/std": 0.3172462284564972, + "rewards/tag_count_reward/mean": 0.94482421875, + "rewards/tag_count_reward/std": 0.17063702642917633, + "step": 1733 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1864.0, + "completions/mean_length": 846.513671875, + "completions/mean_terminated_length": 800.2089233398438, + "completions/min_length": 185.0, + "completions/min_terminated_length": 185.0, + "epoch": 0.5919603994196466, + "grad_norm": 2.042649507522583, + "kl": 5.4140625, + "learning_rate": 4.845596515479113e-07, + "loss": 0.359, + "num_tokens": 961939561.0, + "reward": 1.826171875, + "reward_std": 0.49255073070526123, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.93359375, + "rewards/tag_count_reward/std": 0.18100711703300476, + "step": 1734 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1920.0, + "completions/mean_length": 778.263671875, + "completions/mean_terminated_length": 729.32861328125, + "completions/min_length": 17.0, + "completions/min_terminated_length": 17.0, + "epoch": 0.5923017837330374, + "grad_norm": 1.1241272687911987, + "kl": 5.40625, + "learning_rate": 4.840292892088423e-07, + "loss": 0.3483, + "num_tokens": 962409280.0, + "reward": 1.8349609375, + "reward_std": 0.5005354881286621, + "rewards/accuracy_reward/mean": 0.033203125, + "rewards/accuracy_reward/std": 0.17934183776378632, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.9306640625, + "rewards/tag_count_reward/std": 0.19175048172473907, + "step": 1735 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1998.0, + "completions/mean_length": 864.265625, + "completions/mean_terminated_length": 826.0806274414062, + "completions/min_length": 50.0, + "completions/min_terminated_length": 50.0, + "epoch": 0.5926431680464282, + "grad_norm": 2.7050602436065674, + "kl": 6.8671875, + "learning_rate": 4.834990205031314e-07, + "loss": 0.424, + "num_tokens": 962925720.0, + "reward": 1.84765625, + "reward_std": 0.5242334604263306, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.919921875, + "rewards/tag_count_reward/std": 0.19767135381698608, + "step": 1736 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 806.775390625, + "completions/mean_terminated_length": 761.548583984375, + "completions/min_length": 143.0, + "completions/min_terminated_length": 143.0, + "epoch": 0.5929845523598191, + "grad_norm": 1.4915874004364014, + "kl": 6.6875, + "learning_rate": 4.829688461833975e-07, + "loss": 0.4136, + "num_tokens": 963409205.0, + "reward": 1.8388671875, + "reward_std": 0.5788981914520264, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.9150390625, + "rewards/tag_count_reward/std": 0.21240492165088654, + "step": 1737 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 789.794921875, + "completions/mean_terminated_length": 741.3042602539062, + "completions/min_length": 166.0, + "completions/min_terminated_length": 166.0, + "epoch": 0.5933259366732099, + "grad_norm": 1.7969499826431274, + "kl": 6.6875, + "learning_rate": 4.824387670021263e-07, + "loss": 0.4079, + "num_tokens": 963888348.0, + "reward": 1.88232421875, + "reward_std": 0.504429042339325, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.8671875, + "rewards/format_reward/std": 0.33970388770103455, + "rewards/tag_count_reward/mean": 0.93115234375, + "rewards/tag_count_reward/std": 0.19224505126476288, + "step": 1738 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 800.53125, + "completions/mean_terminated_length": 765.4617919921875, + "completions/min_length": 166.0, + "completions/min_terminated_length": 166.0, + "epoch": 0.5936673209866007, + "grad_norm": 1.737382173538208, + "kl": 6.515625, + "learning_rate": 4.819087837116682e-07, + "loss": 0.3886, + "num_tokens": 964372828.0, + "reward": 1.83642578125, + "reward_std": 0.5354998707771301, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.92431640625, + "rewards/tag_count_reward/std": 0.1941147744655609, + "step": 1739 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 753.48046875, + "completions/mean_terminated_length": 709.0222778320312, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "epoch": 0.5940087052999915, + "grad_norm": 3.20028018951416, + "kl": 7.2890625, + "learning_rate": 4.813788970642373e-07, + "loss": 0.4455, + "num_tokens": 964836610.0, + "reward": 1.8701171875, + "reward_std": 0.5288277864456177, + "rewards/accuracy_reward/mean": 0.07661290466785431, + "rewards/accuracy_reward/std": 0.2662447690963745, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.9248046875, + "rewards/tag_count_reward/std": 0.20080552995204926, + "step": 1740 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2002.0, + "completions/mean_length": 794.53125, + "completions/mean_terminated_length": 756.7001953125, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "epoch": 0.5943500896133823, + "grad_norm": 3.0313708782196045, + "kl": 7.06640625, + "learning_rate": 4.808491078119114e-07, + "loss": 0.4024, + "num_tokens": 965320258.0, + "reward": 1.86328125, + "reward_std": 0.49829375743865967, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.859375, + "rewards/format_reward/std": 0.3479743003845215, + "rewards/tag_count_reward/mean": 0.927734375, + "rewards/tag_count_reward/std": 0.18937622010707855, + "step": 1741 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1989.0, + "completions/mean_length": 862.416015625, + "completions/mean_terminated_length": 811.7088012695312, + "completions/min_length": 89.0, + "completions/min_terminated_length": 89.0, + "epoch": 0.594691473926773, + "grad_norm": 1.8369535207748413, + "kl": 7.1328125, + "learning_rate": 4.803194167066292e-07, + "loss": 0.4815, + "num_tokens": 965840903.0, + "reward": 1.8203125, + "reward_std": 0.5239598751068115, + "rewards/accuracy_reward/mean": 0.029296875, + "rewards/accuracy_reward/std": 0.16880230605602264, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.9296875, + "rewards/tag_count_reward/std": 0.18817149102687836, + "step": 1742 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2010.0, + "completions/mean_length": 802.103515625, + "completions/mean_terminated_length": 774.7484741210938, + "completions/min_length": 149.0, + "completions/min_terminated_length": 149.0, + "epoch": 0.5950328582401638, + "grad_norm": 2.128993034362793, + "kl": 6.3984375, + "learning_rate": 4.7978982450019e-07, + "loss": 0.3625, + "num_tokens": 966318460.0, + "reward": 1.8330078125, + "reward_std": 0.5399568676948547, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.9248046875, + "rewards/tag_count_reward/std": 0.18822988867759705, + "step": 1743 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 804.224609375, + "completions/mean_terminated_length": 779.4482421875, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "epoch": 0.5953742425535546, + "grad_norm": 2.5650181770324707, + "kl": 4.31640625, + "learning_rate": 4.792603319442533e-07, + "loss": 0.2638, + "num_tokens": 966807199.0, + "reward": 1.83935546875, + "reward_std": 0.4204694628715515, + "rewards/accuracy_reward/mean": 0.025390625, + "rewards/accuracy_reward/std": 0.15746226906776428, + "rewards/format_reward/mean": 0.873046875, + "rewards/format_reward/std": 0.33324605226516724, + "rewards/tag_count_reward/mean": 0.94091796875, + "rewards/tag_count_reward/std": 0.1678706705570221, + "step": 1744 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1974.0, + "completions/mean_length": 825.75390625, + "completions/mean_terminated_length": 796.4200439453125, + "completions/min_length": 193.0, + "completions/min_terminated_length": 193.0, + "epoch": 0.5957156268669455, + "grad_norm": 2.448404312133789, + "kl": 4.71875, + "learning_rate": 4.78730939790337e-07, + "loss": 0.2983, + "num_tokens": 967309873.0, + "reward": 1.88525390625, + "reward_std": 0.4958968460559845, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.88671875, + "rewards/format_reward/std": 0.3172462284564972, + "rewards/tag_count_reward/mean": 0.93798828125, + "rewards/tag_count_reward/std": 0.17677602171897888, + "step": 1745 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2024.0, + "completions/mean_length": 883.736328125, + "completions/mean_terminated_length": 843.7515258789062, + "completions/min_length": 146.0, + "completions/min_terminated_length": 146.0, + "epoch": 0.5960570111803363, + "grad_norm": 2.7602598667144775, + "kl": 5.578125, + "learning_rate": 4.782016487898163e-07, + "loss": 0.3717, + "num_tokens": 967838458.0, + "reward": 1.8173828125, + "reward_std": 0.5011401772499084, + "rewards/accuracy_reward/mean": 0.03125, + "rewards/accuracy_reward/std": 0.17416280508041382, + "rewards/format_reward/mean": 0.859375, + "rewards/format_reward/std": 0.3479743003845215, + "rewards/tag_count_reward/mean": 0.9267578125, + "rewards/tag_count_reward/std": 0.18835169076919556, + "step": 1746 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2031.0, + "completions/mean_length": 852.3359375, + "completions/mean_terminated_length": 823.6400146484375, + "completions/min_length": 211.0, + "completions/min_terminated_length": 211.0, + "epoch": 0.5963983954937271, + "grad_norm": 1.6309069395065308, + "kl": 4.685546875, + "learning_rate": 4.77672459693923e-07, + "loss": 0.308, + "num_tokens": 968354118.0, + "reward": 1.91455078125, + "reward_std": 0.46543601155281067, + "rewards/accuracy_reward/mean": 0.08467742055654526, + "rewards/accuracy_reward/std": 0.278682142496109, + "rewards/format_reward/mean": 0.890625, + "rewards/format_reward/std": 0.31241437792778015, + "rewards/tag_count_reward/mean": 0.94189453125, + "rewards/tag_count_reward/std": 0.1739315241575241, + "step": 1747 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1915.0, + "completions/mean_length": 782.896484375, + "completions/mean_terminated_length": 755.1197509765625, + "completions/min_length": 170.0, + "completions/min_terminated_length": 170.0, + "epoch": 0.5967397798071179, + "grad_norm": 1.6585683822631836, + "kl": 4.765625, + "learning_rate": 4.771433732537446e-07, + "loss": 0.3026, + "num_tokens": 968830913.0, + "reward": 1.88818359375, + "reward_std": 0.4815041720867157, + "rewards/accuracy_reward/mean": 0.08266129344701767, + "rewards/accuracy_reward/std": 0.2756475806236267, + "rewards/format_reward/mean": 0.876953125, + "rewards/format_reward/std": 0.32881227135658264, + "rewards/tag_count_reward/mean": 0.93115234375, + "rewards/tag_count_reward/std": 0.1909683793783188, + "step": 1748 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1977.0, + "completions/mean_length": 831.87109375, + "completions/mean_terminated_length": 756.178466796875, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "epoch": 0.5970811641205087, + "grad_norm": 1.4092459678649902, + "kl": 8.4375, + "learning_rate": 4.7661439022022186e-07, + "loss": 0.5519, + "num_tokens": 969329967.0, + "reward": 1.81201171875, + "reward_std": 0.527290940284729, + "rewards/accuracy_reward/mean": 0.04032257944345474, + "rewards/accuracy_reward/std": 0.19691328704357147, + "rewards/format_reward/mean": 0.849609375, + "rewards/format_reward/std": 0.35780346393585205, + "rewards/tag_count_reward/mean": 0.92333984375, + "rewards/tag_count_reward/std": 0.19871696829795837, + "step": 1749 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1996.0, + "completions/mean_length": 828.638671875, + "completions/mean_terminated_length": 794.3594360351562, + "completions/min_length": 41.0, + "completions/min_terminated_length": 41.0, + "epoch": 0.5974225484338994, + "grad_norm": 1.1671388149261475, + "kl": 5.625, + "learning_rate": 4.7608551134415e-07, + "loss": 0.3178, + "num_tokens": 969827126.0, + "reward": 1.87548828125, + "reward_std": 0.545221745967865, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.93212890625, + "rewards/tag_count_reward/std": 0.18809467554092407, + "step": 1750 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1760.0, + "completions/mean_length": 793.0625, + "completions/mean_terminated_length": 736.7183227539062, + "completions/min_length": 165.0, + "completions/min_terminated_length": 165.0, + "epoch": 0.5977639327472902, + "grad_norm": 1.3640391826629639, + "kl": 7.6171875, + "learning_rate": 4.755567373761755e-07, + "loss": 0.4561, + "num_tokens": 970305782.0, + "reward": 1.89111328125, + "reward_std": 0.5875589847564697, + "rewards/accuracy_reward/mean": 0.1171875, + "rewards/accuracy_reward/std": 0.32195815443992615, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.92041015625, + "rewards/tag_count_reward/std": 0.20123985409736633, + "step": 1751 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1870.0, + "completions/mean_length": 852.9453125, + "completions/mean_terminated_length": 799.2897338867188, + "completions/min_length": 119.0, + "completions/min_terminated_length": 119.0, + "epoch": 0.598105317060681, + "grad_norm": 2.3671305179595947, + "kl": 8.265625, + "learning_rate": 4.750280690667965e-07, + "loss": 0.4942, + "num_tokens": 970814026.0, + "reward": 1.8359375, + "reward_std": 0.5786428451538086, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.9140625, + "rewards/tag_count_reward/std": 0.20616121590137482, + "step": 1752 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2020.0, + "completions/mean_length": 788.517578125, + "completions/mean_terminated_length": 745.2626342773438, + "completions/min_length": 99.0, + "completions/min_terminated_length": 99.0, + "epoch": 0.5984467013740719, + "grad_norm": 1.7761777639389038, + "kl": 5.46875, + "learning_rate": 4.744995071663609e-07, + "loss": 0.3671, + "num_tokens": 971298835.0, + "reward": 1.85546875, + "reward_std": 0.5084636807441711, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.8671875, + "rewards/format_reward/std": 0.33970388770103455, + "rewards/tag_count_reward/mean": 0.927734375, + "rewards/tag_count_reward/std": 0.19447441399097443, + "step": 1753 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2005.0, + "completions/mean_length": 809.068359375, + "completions/mean_terminated_length": 771.676025390625, + "completions/min_length": 144.0, + "completions/min_terminated_length": 144.0, + "epoch": 0.5987880856874627, + "grad_norm": 2.005474090576172, + "kl": 6.1640625, + "learning_rate": 4.7397105242506576e-07, + "loss": 0.3522, + "num_tokens": 971784486.0, + "reward": 1.85498046875, + "reward_std": 0.4786653518676758, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.859375, + "rewards/format_reward/std": 0.3479743003845215, + "rewards/tag_count_reward/mean": 0.92529296875, + "rewards/tag_count_reward/std": 0.18679475784301758, + "step": 1754 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2013.0, + "completions/mean_length": 815.5859375, + "completions/mean_terminated_length": 773.2606201171875, + "completions/min_length": 148.0, + "completions/min_terminated_length": 148.0, + "epoch": 0.5991294700008535, + "grad_norm": 1.6871479749679565, + "kl": 5.4296875, + "learning_rate": 4.734427055929556e-07, + "loss": 0.3421, + "num_tokens": 972284898.0, + "reward": 1.9296875, + "reward_std": 0.5219883918762207, + "rewards/accuracy_reward/mean": 0.134765625, + "rewards/accuracy_reward/std": 0.3418070077896118, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.93359375, + "rewards/tag_count_reward/std": 0.17621363699436188, + "step": 1755 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2011.0, + "completions/mean_length": 918.212890625, + "completions/mean_terminated_length": 865.0736083984375, + "completions/min_length": 56.0, + "completions/min_terminated_length": 56.0, + "epoch": 0.5994708543142443, + "grad_norm": 2.2570464611053467, + "kl": 7.0859375, + "learning_rate": 4.729144674199225e-07, + "loss": 0.4195, + "num_tokens": 972835807.0, + "reward": 1.81201171875, + "reward_std": 0.5604456663131714, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.833984375, + "rewards/format_reward/std": 0.3724585771560669, + "rewards/tag_count_reward/mean": 0.91162109375, + "rewards/tag_count_reward/std": 0.21187059581279755, + "step": 1756 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 812.84765625, + "completions/mean_terminated_length": 778.1244506835938, + "completions/min_length": 69.0, + "completions/min_terminated_length": 69.0, + "epoch": 0.5998122386276351, + "grad_norm": 1.5576038360595703, + "kl": 5.0, + "learning_rate": 4.723863386557037e-07, + "loss": 0.2982, + "num_tokens": 973336257.0, + "reward": 1.8232421875, + "reward_std": 0.5296144485473633, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.9208984375, + "rewards/tag_count_reward/std": 0.193694069981575, + "step": 1757 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2016.0, + "completions/mean_length": 863.595703125, + "completions/mean_terminated_length": 797.6598510742188, + "completions/min_length": 72.0, + "completions/min_terminated_length": 72.0, + "epoch": 0.6001536229410258, + "grad_norm": 1.4606642723083496, + "kl": 6.8203125, + "learning_rate": 4.7185832004988133e-07, + "loss": 0.4245, + "num_tokens": 973856546.0, + "reward": 1.79638671875, + "reward_std": 0.584352970123291, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.828125, + "rewards/format_reward/std": 0.3776407241821289, + "rewards/tag_count_reward/mean": 0.89990234375, + "rewards/tag_count_reward/std": 0.2197779268026352, + "step": 1758 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 812.927734375, + "completions/mean_terminated_length": 767.9251098632812, + "completions/min_length": 33.0, + "completions/min_terminated_length": 33.0, + "epoch": 0.6004950072544166, + "grad_norm": 2.716848373413086, + "kl": 6.0625, + "learning_rate": 4.713304123518814e-07, + "loss": 0.3726, + "num_tokens": 974342205.0, + "reward": 1.8486328125, + "reward_std": 0.5314549803733826, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.9228515625, + "rewards/tag_count_reward/std": 0.19321990013122559, + "step": 1759 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1929.0, + "completions/mean_length": 807.177734375, + "completions/mean_terminated_length": 754.1079711914062, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "epoch": 0.6008363915678074, + "grad_norm": 1.1507644653320312, + "kl": 5.33203125, + "learning_rate": 4.708026163109725e-07, + "loss": 0.3146, + "num_tokens": 974835912.0, + "reward": 1.86767578125, + "reward_std": 0.6231339573860168, + "rewards/accuracy_reward/mean": 0.13671875, + "rewards/accuracy_reward/std": 0.3438861668109894, + "rewards/format_reward/mean": 0.8203125, + "rewards/format_reward/std": 0.38430243730545044, + "rewards/tag_count_reward/mean": 0.91064453125, + "rewards/tag_count_reward/std": 0.20320084691047668, + "step": 1760 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1924.0, + "completions/mean_length": 762.54296875, + "completions/mean_terminated_length": 731.6920166015625, + "completions/min_length": 175.0, + "completions/min_terminated_length": 175.0, + "epoch": 0.6011777758811983, + "grad_norm": 1.9752110242843628, + "kl": 3.89453125, + "learning_rate": 4.7027493267626405e-07, + "loss": 0.2471, + "num_tokens": 975302254.0, + "reward": 1.9130859375, + "reward_std": 0.4799373745918274, + "rewards/accuracy_reward/mean": 0.099609375, + "rewards/accuracy_reward/std": 0.29977133870124817, + "rewards/format_reward/mean": 0.875, + "rewards/format_reward/std": 0.3310423493385315, + "rewards/tag_count_reward/mean": 0.9384765625, + "rewards/tag_count_reward/std": 0.1709705889225006, + "step": 1761 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 856.0390625, + "completions/mean_terminated_length": 815.1030883789062, + "completions/min_length": 150.0, + "completions/min_terminated_length": 150.0, + "epoch": 0.6015191601945891, + "grad_norm": 1.9344838857650757, + "kl": 4.41015625, + "learning_rate": 4.69747362196707e-07, + "loss": 0.2692, + "num_tokens": 975815538.0, + "reward": 1.84375, + "reward_std": 0.5603270530700684, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.916015625, + "rewards/tag_count_reward/std": 0.20338943600654602, + "step": 1762 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1928.0, + "completions/mean_length": 765.466796875, + "completions/mean_terminated_length": 739.9183349609375, + "completions/min_length": 221.0, + "completions/min_terminated_length": 221.0, + "epoch": 0.6018605445079799, + "grad_norm": 2.057072401046753, + "kl": 4.1875, + "learning_rate": 4.692199056210907e-07, + "loss": 0.2883, + "num_tokens": 976278129.0, + "reward": 1.89599609375, + "reward_std": 0.4938367009162903, + "rewards/accuracy_reward/mean": 0.08870967477560043, + "rewards/accuracy_reward/std": 0.284611314535141, + "rewards/format_reward/mean": 0.880859375, + "rewards/format_reward/std": 0.32427072525024414, + "rewards/tag_count_reward/mean": 0.92919921875, + "rewards/tag_count_reward/std": 0.18700948357582092, + "step": 1763 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2007.0, + "completions/mean_length": 817.630859375, + "completions/mean_terminated_length": 780.4969482421875, + "completions/min_length": 151.0, + "completions/min_terminated_length": 151.0, + "epoch": 0.6022019288213707, + "grad_norm": 2.9331674575805664, + "kl": 4.86328125, + "learning_rate": 4.6869256369804353e-07, + "loss": 0.3337, + "num_tokens": 976778484.0, + "reward": 1.8681640625, + "reward_std": 0.5320004224777222, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.8671875, + "rewards/format_reward/std": 0.33970388770103455, + "rewards/tag_count_reward/mean": 0.9326171875, + "rewards/tag_count_reward/std": 0.1779162585735321, + "step": 1764 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1985.0, + "completions/mean_length": 834.357421875, + "completions/mean_terminated_length": 785.0223388671875, + "completions/min_length": 141.0, + "completions/min_terminated_length": 141.0, + "epoch": 0.6025433131347615, + "grad_norm": 1.946372628211975, + "kl": 5.8671875, + "learning_rate": 4.6816533717603093e-07, + "loss": 0.3678, + "num_tokens": 977282603.0, + "reward": 1.822265625, + "reward_std": 0.4897192716598511, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.923828125, + "rewards/tag_count_reward/std": 0.19042283296585083, + "step": 1765 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 851.9609375, + "completions/mean_terminated_length": 785.3773193359375, + "completions/min_length": 182.0, + "completions/min_terminated_length": 182.0, + "epoch": 0.6028846974481522, + "grad_norm": 2.542891025543213, + "kl": 8.015625, + "learning_rate": 4.676382268033544e-07, + "loss": 0.4818, + "num_tokens": 977793479.0, + "reward": 1.78271484375, + "reward_std": 0.5828242897987366, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.806640625, + "rewards/format_reward/std": 0.39531853795051575, + "rewards/tag_count_reward/mean": 0.90771484375, + "rewards/tag_count_reward/std": 0.21019525825977325, + "step": 1766 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1979.0, + "completions/mean_length": 872.7109375, + "completions/mean_terminated_length": 812.3778686523438, + "completions/min_length": 118.0, + "completions/min_terminated_length": 118.0, + "epoch": 0.603226081761543, + "grad_norm": 3.6636009216308594, + "kl": 8.1875, + "learning_rate": 4.671112333281508e-07, + "loss": 0.472, + "num_tokens": 978319091.0, + "reward": 1.7412109375, + "reward_std": 0.6191123127937317, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.787109375, + "rewards/format_reward/std": 0.409751296043396, + "rewards/tag_count_reward/mean": 0.8916015625, + "rewards/tag_count_reward/std": 0.22221922874450684, + "step": 1767 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1977.0, + "completions/mean_length": 807.677734375, + "completions/mean_terminated_length": 759.876220703125, + "completions/min_length": 224.0, + "completions/min_terminated_length": 224.0, + "epoch": 0.6035674660749338, + "grad_norm": 1.4426823854446411, + "kl": 6.58984375, + "learning_rate": 4.6658435749839087e-07, + "loss": 0.3797, + "num_tokens": 978810686.0, + "reward": 1.8359375, + "reward_std": 0.5612466335296631, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.841796875, + "rewards/format_reward/std": 0.36528825759887695, + "rewards/tag_count_reward/mean": 0.921875, + "rewards/tag_count_reward/std": 0.1972164362668991, + "step": 1768 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1993.0, + "completions/mean_length": 808.4453125, + "completions/mean_terminated_length": 768.4596557617188, + "completions/min_length": 181.0, + "completions/min_terminated_length": 181.0, + "epoch": 0.6039088503883246, + "grad_norm": 3.0837481021881104, + "kl": 8.3828125, + "learning_rate": 4.6605760006187857e-07, + "loss": 0.4804, + "num_tokens": 979296594.0, + "reward": 1.814453125, + "reward_std": 0.5509793758392334, + "rewards/accuracy_reward/mean": 0.09677419066429138, + "rewards/accuracy_reward/std": 0.2959485352039337, + "rewards/format_reward/mean": 0.80859375, + "rewards/format_reward/std": 0.3937928080558777, + "rewards/tag_count_reward/mean": 0.912109375, + "rewards/tag_count_reward/std": 0.19990174472332, + "step": 1769 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 833.59375, + "completions/mean_terminated_length": 789.3441772460938, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "epoch": 0.6042502347017155, + "grad_norm": 1.2210311889648438, + "kl": 6.515625, + "learning_rate": 4.655309617662496e-07, + "loss": 0.3899, + "num_tokens": 979796498.0, + "reward": 1.78857421875, + "reward_std": 0.5330498814582825, + "rewards/accuracy_reward/mean": 0.04032257944345474, + "rewards/accuracy_reward/std": 0.19691328704357147, + "rewards/format_reward/mean": 0.830078125, + "rewards/format_reward/std": 0.3759314715862274, + "rewards/tag_count_reward/mean": 0.91943359375, + "rewards/tag_count_reward/std": 0.19277119636535645, + "step": 1770 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 808.107421875, + "completions/mean_terminated_length": 765.5252685546875, + "completions/min_length": 5.0, + "completions/min_terminated_length": 5.0, + "epoch": 0.6045916190151063, + "grad_norm": 1.361059308052063, + "kl": 6.078125, + "learning_rate": 4.6500444335897094e-07, + "loss": 0.379, + "num_tokens": 980292009.0, + "reward": 1.8291015625, + "reward_std": 0.5372997522354126, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.9287109375, + "rewards/tag_count_reward/std": 0.18845312297344208, + "step": 1771 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 869.541015625, + "completions/mean_terminated_length": 826.6012573242188, + "completions/min_length": 173.0, + "completions/min_terminated_length": 173.0, + "epoch": 0.6049330033284971, + "grad_norm": 0.9149605631828308, + "kl": 7.703125, + "learning_rate": 4.6447804558733894e-07, + "loss": 0.4712, + "num_tokens": 980813406.0, + "reward": 1.7626953125, + "reward_std": 0.5920639038085938, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.8046875, + "rewards/format_reward/std": 0.3968288004398346, + "rewards/tag_count_reward/mean": 0.9072265625, + "rewards/tag_count_reward/std": 0.21200865507125854, + "step": 1772 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 873.810546875, + "completions/mean_terminated_length": 828.5578002929688, + "completions/min_length": 153.0, + "completions/min_terminated_length": 153.0, + "epoch": 0.6052743876418879, + "grad_norm": 1.3184945583343506, + "kl": 6.984375, + "learning_rate": 4.6395176919847923e-07, + "loss": 0.4523, + "num_tokens": 981341549.0, + "reward": 1.8056640625, + "reward_std": 0.529266893863678, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.833984375, + "rewards/format_reward/std": 0.3724585771560669, + "rewards/tag_count_reward/mean": 0.9267578125, + "rewards/tag_count_reward/std": 0.18835169076919556, + "step": 1773 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 920.623046875, + "completions/mean_terminated_length": 865.17822265625, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "epoch": 0.6056157719552786, + "grad_norm": 0.7841996550559998, + "kl": 7.484375, + "learning_rate": 4.634256149393445e-07, + "loss": 0.4723, + "num_tokens": 981891260.0, + "reward": 1.80615234375, + "reward_std": 0.5783164501190186, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.833984375, + "rewards/format_reward/std": 0.3724585771560669, + "rewards/tag_count_reward/mean": 0.91162109375, + "rewards/tag_count_reward/std": 0.2175668478012085, + "step": 1774 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2007.0, + "completions/mean_length": 841.51171875, + "completions/mean_terminated_length": 787.3428344726562, + "completions/min_length": 70.0, + "completions/min_terminated_length": 70.0, + "epoch": 0.6059571562686694, + "grad_norm": 2.017267942428589, + "kl": 8.0546875, + "learning_rate": 4.6289958355671475e-07, + "loss": 0.5094, + "num_tokens": 982402210.0, + "reward": 1.7998046875, + "reward_std": 0.5805681943893433, + "rewards/accuracy_reward/mean": 0.06854838877916336, + "rewards/accuracy_reward/std": 0.25293973088264465, + "rewards/format_reward/mean": 0.826171875, + "rewards/format_reward/std": 0.3793322443962097, + "rewards/tag_count_reward/mean": 0.9072265625, + "rewards/tag_count_reward/std": 0.21713875234127045, + "step": 1775 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1973.0, + "completions/mean_length": 795.16015625, + "completions/mean_terminated_length": 733.5450439453125, + "completions/min_length": 139.0, + "completions/min_terminated_length": 139.0, + "epoch": 0.6062985405820602, + "grad_norm": 1.6451234817504883, + "kl": 7.171875, + "learning_rate": 4.6237367579719535e-07, + "loss": 0.4567, + "num_tokens": 982886500.0, + "reward": 1.84814453125, + "reward_std": 0.5437964200973511, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.857421875, + "rewards/format_reward/std": 0.3499840497970581, + "rewards/tag_count_reward/mean": 0.92236328125, + "rewards/tag_count_reward/std": 0.20619770884513855, + "step": 1776 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.056640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 888.806640625, + "completions/mean_terminated_length": 819.20703125, + "completions/min_length": 215.0, + "completions/min_terminated_length": 215.0, + "epoch": 0.606639924895451, + "grad_norm": 2.459740161895752, + "kl": 8.1015625, + "learning_rate": 4.618478924072159e-07, + "loss": 0.5034, + "num_tokens": 983413681.0, + "reward": 1.826171875, + "reward_std": 0.5603066086769104, + "rewards/accuracy_reward/mean": 0.08064515888690948, + "rewards/accuracy_reward/std": 0.2725643217563629, + "rewards/format_reward/mean": 0.833984375, + "rewards/format_reward/std": 0.3724585771560669, + "rewards/tag_count_reward/mean": 0.9140625, + "rewards/tag_count_reward/std": 0.21027308702468872, + "step": 1777 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2014.0, + "completions/mean_length": 778.66796875, + "completions/mean_terminated_length": 721.677490234375, + "completions/min_length": 123.0, + "completions/min_terminated_length": 123.0, + "epoch": 0.6069813092088419, + "grad_norm": 2.2952628135681152, + "kl": 5.4375, + "learning_rate": 4.6132223413303e-07, + "loss": 0.369, + "num_tokens": 983888807.0, + "reward": 1.88818359375, + "reward_std": 0.5219037532806396, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.875, + "rewards/format_reward/std": 0.3310423493385315, + "rewards/tag_count_reward/mean": 0.93310546875, + "rewards/tag_count_reward/std": 0.18844488263130188, + "step": 1778 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1951.0, + "completions/mean_length": 869.060546875, + "completions/mean_terminated_length": 803.4288940429688, + "completions/min_length": 154.0, + "completions/min_terminated_length": 154.0, + "epoch": 0.6073226935222327, + "grad_norm": 1.1419627666473389, + "kl": 6.7265625, + "learning_rate": 4.607967017207135e-07, + "loss": 0.4287, + "num_tokens": 984408246.0, + "reward": 1.82421875, + "reward_std": 0.5274852514266968, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.91796875, + "rewards/tag_count_reward/std": 0.20715993642807007, + "step": 1779 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0703125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2017.0, + "completions/mean_length": 941.478515625, + "completions/mean_terminated_length": 857.7920532226562, + "completions/min_length": 19.0, + "completions/min_terminated_length": 19.0, + "epoch": 0.6076640778356235, + "grad_norm": 1.2240148782730103, + "kl": 8.890625, + "learning_rate": 4.602712959161633e-07, + "loss": 0.5813, + "num_tokens": 984972603.0, + "reward": 1.72705078125, + "reward_std": 0.6361920833587646, + "rewards/accuracy_reward/mean": 0.037109375, + "rewards/accuracy_reward/std": 0.18921469151973724, + "rewards/format_reward/mean": 0.798828125, + "rewards/format_reward/std": 0.4012683033943176, + "rewards/tag_count_reward/mean": 0.89111328125, + "rewards/tag_count_reward/std": 0.23404912650585175, + "step": 1780 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2009.0, + "completions/mean_length": 802.990234375, + "completions/mean_terminated_length": 760.2323608398438, + "completions/min_length": 64.0, + "completions/min_terminated_length": 64.0, + "epoch": 0.6080054621490143, + "grad_norm": 2.643549919128418, + "kl": 5.59375, + "learning_rate": 4.5974601746509687e-07, + "loss": 0.3628, + "num_tokens": 985459318.0, + "reward": 1.85791015625, + "reward_std": 0.5174456834793091, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.93212890625, + "rewards/tag_count_reward/std": 0.1944885402917862, + "step": 1781 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1951.0, + "completions/mean_length": 801.52734375, + "completions/mean_terminated_length": 776.6972045898438, + "completions/min_length": 36.0, + "completions/min_terminated_length": 36.0, + "epoch": 0.608346846462405, + "grad_norm": 2.1473066806793213, + "kl": 5.1171875, + "learning_rate": 4.592208671130511e-07, + "loss": 0.3116, + "num_tokens": 985940276.0, + "reward": 1.880859375, + "reward_std": 0.4827001094818115, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.857421875, + "rewards/format_reward/std": 0.3499840497970581, + "rewards/tag_count_reward/mean": 0.939453125, + "rewards/tag_count_reward/std": 0.16916421055793762, + "step": 1782 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2030.0, + "completions/mean_length": 885.35546875, + "completions/mean_terminated_length": 825.6715087890625, + "completions/min_length": 201.0, + "completions/min_terminated_length": 201.0, + "epoch": 0.6086882307757958, + "grad_norm": 1.3922321796417236, + "kl": 8.65625, + "learning_rate": 4.5869584560538065e-07, + "loss": 0.5819, + "num_tokens": 986471658.0, + "reward": 1.8017578125, + "reward_std": 0.5646194815635681, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.8359375, + "rewards/format_reward/std": 0.37069445848464966, + "rewards/tag_count_reward/mean": 0.9169921875, + "rewards/tag_count_reward/std": 0.20736047625541687, + "step": 1783 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1966.0, + "completions/mean_length": 896.87109375, + "completions/mean_terminated_length": 835.2880249023438, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "epoch": 0.6090296150891866, + "grad_norm": 2.539964437484741, + "kl": 7.078125, + "learning_rate": 4.5817095368725754e-07, + "loss": 0.4367, + "num_tokens": 987017864.0, + "reward": 1.81103515625, + "reward_std": 0.5181088447570801, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.92041015625, + "rewards/tag_count_reward/std": 0.19693946838378906, + "step": 1784 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2022.0, + "completions/mean_length": 829.134765625, + "completions/mean_terminated_length": 771.8057250976562, + "completions/min_length": 181.0, + "completions/min_terminated_length": 181.0, + "epoch": 0.6093709994025774, + "grad_norm": 1.532462239265442, + "kl": 6.3046875, + "learning_rate": 4.576461921036702e-07, + "loss": 0.373, + "num_tokens": 987520045.0, + "reward": 1.837890625, + "reward_std": 0.5260030031204224, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.923828125, + "rewards/tag_count_reward/std": 0.18913386762142181, + "step": 1785 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1911.0, + "completions/mean_length": 769.2421875, + "completions/mean_terminated_length": 730.6478881835938, + "completions/min_length": 9.0, + "completions/min_terminated_length": 9.0, + "epoch": 0.6097123837159683, + "grad_norm": 1.843642234802246, + "kl": 6.5234375, + "learning_rate": 4.571215615994212e-07, + "loss": 0.3969, + "num_tokens": 987992569.0, + "reward": 1.791015625, + "reward_std": 0.515088677406311, + "rewards/accuracy_reward/mean": 0.021484375, + "rewards/accuracy_reward/std": 0.14513419568538666, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.91796875, + "rewards/tag_count_reward/std": 0.2005603015422821, + "step": 1786 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1993.0, + "completions/mean_length": 830.81640625, + "completions/mean_terminated_length": 778.7576904296875, + "completions/min_length": 108.0, + "completions/min_terminated_length": 108.0, + "epoch": 0.6100537680293591, + "grad_norm": 2.2469699382781982, + "kl": 7.3203125, + "learning_rate": 4.565970629191278e-07, + "loss": 0.4796, + "num_tokens": 988498443.0, + "reward": 1.83935546875, + "reward_std": 0.5874500274658203, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.841796875, + "rewards/format_reward/std": 0.36528825759887695, + "rewards/tag_count_reward/mean": 0.91357421875, + "rewards/tag_count_reward/std": 0.20386749505996704, + "step": 1787 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2030.0, + "completions/mean_length": 836.029296875, + "completions/mean_terminated_length": 773.8131713867188, + "completions/min_length": 89.0, + "completions/min_terminated_length": 89.0, + "epoch": 0.6103951523427499, + "grad_norm": 1.9404956102371216, + "kl": 6.78125, + "learning_rate": 4.5607269680721993e-07, + "loss": 0.4275, + "num_tokens": 989004986.0, + "reward": 1.83154296875, + "reward_std": 0.5891203880310059, + "rewards/accuracy_reward/mean": 0.08467742055654526, + "rewards/accuracy_reward/std": 0.278682142496109, + "rewards/format_reward/mean": 0.833984375, + "rewards/format_reward/std": 0.3724585771560669, + "rewards/tag_count_reward/mean": 0.91552734375, + "rewards/tag_count_reward/std": 0.20348748564720154, + "step": 1788 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2012.0, + "completions/mean_length": 855.521484375, + "completions/mean_terminated_length": 812.0708618164062, + "completions/min_length": 155.0, + "completions/min_terminated_length": 155.0, + "epoch": 0.6107365366561407, + "grad_norm": 3.4830024242401123, + "kl": 5.78125, + "learning_rate": 4.5554846400793946e-07, + "loss": 0.3474, + "num_tokens": 989519189.0, + "reward": 1.8271484375, + "reward_std": 0.524437427520752, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.841796875, + "rewards/format_reward/std": 0.36528825759887695, + "rewards/tag_count_reward/mean": 0.9150390625, + "rewards/tag_count_reward/std": 0.20418420433998108, + "step": 1789 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1979.0, + "completions/mean_length": 869.517578125, + "completions/mean_terminated_length": 819.1140747070312, + "completions/min_length": 145.0, + "completions/min_terminated_length": 145.0, + "epoch": 0.6110779209695314, + "grad_norm": 1.4256885051727295, + "kl": 5.33984375, + "learning_rate": 4.5502436526533896e-07, + "loss": 0.3369, + "num_tokens": 990048414.0, + "reward": 1.81103515625, + "reward_std": 0.540465235710144, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.91650390625, + "rewards/tag_count_reward/std": 0.2020832896232605, + "step": 1790 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 830.203125, + "completions/mean_terminated_length": 783.269775390625, + "completions/min_length": 84.0, + "completions/min_terminated_length": 84.0, + "epoch": 0.6114193052829222, + "grad_norm": 1.4382879734039307, + "kl": 6.2109375, + "learning_rate": 4.5450040132328074e-07, + "loss": 0.3999, + "num_tokens": 990549542.0, + "reward": 1.818359375, + "reward_std": 0.5191929340362549, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.857421875, + "rewards/format_reward/std": 0.3499840497970581, + "rewards/tag_count_reward/mean": 0.9140625, + "rewards/tag_count_reward/std": 0.20497123897075653, + "step": 1791 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2019.0, + "completions/mean_length": 764.9296875, + "completions/mean_terminated_length": 744.5635375976562, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "epoch": 0.611760689596313, + "grad_norm": 3.7264513969421387, + "kl": 4.953125, + "learning_rate": 4.539765729254356e-07, + "loss": 0.3689, + "num_tokens": 991009922.0, + "reward": 1.89794921875, + "reward_std": 0.5064840912818909, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.888671875, + "rewards/format_reward/std": 0.31484565138816833, + "rewards/tag_count_reward/mean": 0.94287109375, + "rewards/tag_count_reward/std": 0.16781944036483765, + "step": 1792 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1991.0, + "completions/mean_length": 868.408203125, + "completions/mean_terminated_length": 812.9263916015625, + "completions/min_length": 163.0, + "completions/min_terminated_length": 163.0, + "epoch": 0.6121020739097038, + "grad_norm": 2.518594264984131, + "kl": 5.83984375, + "learning_rate": 4.5345288081528223e-07, + "loss": 0.4031, + "num_tokens": 991533651.0, + "reward": 1.77587890625, + "reward_std": 0.545759916305542, + "rewards/accuracy_reward/mean": 0.03125, + "rewards/accuracy_reward/std": 0.17416280508041382, + "rewards/format_reward/mean": 0.830078125, + "rewards/format_reward/std": 0.3759314715862274, + "rewards/tag_count_reward/mean": 0.91455078125, + "rewards/tag_count_reward/std": 0.20959889888763428, + "step": 1793 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1923.0, + "completions/mean_length": 859.126953125, + "completions/mean_terminated_length": 790.34912109375, + "completions/min_length": 145.0, + "completions/min_terminated_length": 145.0, + "epoch": 0.6124434582230946, + "grad_norm": 1.2283124923706055, + "kl": 6.4296875, + "learning_rate": 4.529293257361059e-07, + "loss": 0.3984, + "num_tokens": 992054148.0, + "reward": 1.79736328125, + "reward_std": 0.5642428994178772, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.826171875, + "rewards/format_reward/std": 0.3793322443962097, + "rewards/tag_count_reward/mean": 0.90478515625, + "rewards/tag_count_reward/std": 0.21916839480400085, + "step": 1794 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 761.9375, + "completions/mean_terminated_length": 725.7830810546875, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "epoch": 0.6127848425364855, + "grad_norm": 1.0940067768096924, + "kl": 4.1796875, + "learning_rate": 4.5240590843099725e-07, + "loss": 0.2422, + "num_tokens": 992532868.0, + "reward": 1.8994140625, + "reward_std": 0.4732905924320221, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.876953125, + "rewards/format_reward/std": 0.32881227135658264, + "rewards/tag_count_reward/mean": 0.9384765625, + "rewards/tag_count_reward/std": 0.17729215323925018, + "step": 1795 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1960.0, + "completions/mean_length": 836.771484375, + "completions/mean_terminated_length": 782.3897705078125, + "completions/min_length": 82.0, + "completions/min_terminated_length": 82.0, + "epoch": 0.6131262268498763, + "grad_norm": 1.637098789215088, + "kl": 5.5546875, + "learning_rate": 4.5188262964285126e-07, + "loss": 0.3707, + "num_tokens": 993043439.0, + "reward": 1.85009765625, + "reward_std": 0.4896104633808136, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.93603515625, + "rewards/tag_count_reward/std": 0.18488718569278717, + "step": 1796 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1974.0, + "completions/mean_length": 796.544921875, + "completions/mean_terminated_length": 737.6829833984375, + "completions/min_length": 233.0, + "completions/min_terminated_length": 233.0, + "epoch": 0.6134676111632671, + "grad_norm": 1.0109014511108398, + "kl": 7.0390625, + "learning_rate": 4.513594901143668e-07, + "loss": 0.4573, + "num_tokens": 993528486.0, + "reward": 1.8330078125, + "reward_std": 0.4917706251144409, + "rewards/accuracy_reward/mean": 0.037109375, + "rewards/accuracy_reward/std": 0.18921469151973724, + "rewards/format_reward/mean": 0.86328125, + "rewards/format_reward/std": 0.3438861668109894, + "rewards/tag_count_reward/mean": 0.9326171875, + "rewards/tag_count_reward/std": 0.18266506493091583, + "step": 1797 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2025.0, + "completions/mean_length": 802.662109375, + "completions/mean_terminated_length": 757.2854614257812, + "completions/min_length": 143.0, + "completions/min_terminated_length": 143.0, + "epoch": 0.6138089954766578, + "grad_norm": 1.6239712238311768, + "kl": 5.8203125, + "learning_rate": 4.508364905880444e-07, + "loss": 0.3338, + "num_tokens": 994030281.0, + "reward": 1.8154296875, + "reward_std": 0.4730170965194702, + "rewards/accuracy_reward/mean": 0.033203125, + "rewards/accuracy_reward/std": 0.17934183776378632, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.9267578125, + "rewards/tag_count_reward/std": 0.19093148410320282, + "step": 1798 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 836.919921875, + "completions/mean_terminated_length": 790.2454223632812, + "completions/min_length": 149.0, + "completions/min_terminated_length": 149.0, + "epoch": 0.6141503797900486, + "grad_norm": 4.2510151863098145, + "kl": 9.7265625, + "learning_rate": 4.503136318061863e-07, + "loss": 0.6063, + "num_tokens": 994535536.0, + "reward": 1.7734375, + "reward_std": 0.5860233306884766, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.81640625, + "rewards/format_reward/std": 0.3875311613082886, + "rewards/tag_count_reward/mean": 0.904296875, + "rewards/tag_count_reward/std": 0.2203473001718521, + "step": 1799 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1873.0, + "completions/mean_length": 739.123046875, + "completions/mean_terminated_length": 699.6196899414062, + "completions/min_length": 155.0, + "completions/min_terminated_length": 155.0, + "epoch": 0.6144917641034394, + "grad_norm": 2.4480273723602295, + "kl": 7.953125, + "learning_rate": 4.497909145108949e-07, + "loss": 0.4878, + "num_tokens": 994994879.0, + "reward": 1.828125, + "reward_std": 0.5364860892295837, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.849609375, + "rewards/format_reward/std": 0.35780346393585205, + "rewards/tag_count_reward/mean": 0.92578125, + "rewards/tag_count_reward/std": 0.1931036412715912, + "step": 1800 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1919.0, + "completions/mean_length": 746.984375, + "completions/mean_terminated_length": 713.0902099609375, + "completions/min_length": 18.0, + "completions/min_terminated_length": 18.0, + "epoch": 0.6148331484168302, + "grad_norm": 3.0909080505371094, + "kl": 6.66796875, + "learning_rate": 4.4926833944407207e-07, + "loss": 0.3548, + "num_tokens": 995446583.0, + "reward": 1.8310546875, + "reward_std": 0.5081585645675659, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.9267578125, + "rewards/tag_count_reward/std": 0.18964596092700958, + "step": 1801 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 723.724609375, + "completions/mean_terminated_length": 675.4716796875, + "completions/min_length": 160.0, + "completions/min_terminated_length": 160.0, + "epoch": 0.615174532730221, + "grad_norm": 2.3278188705444336, + "kl": 6.2265625, + "learning_rate": 4.4874590734741715e-07, + "loss": 0.3666, + "num_tokens": 995890890.0, + "reward": 1.9013671875, + "reward_std": 0.5298936367034912, + "rewards/accuracy_reward/mean": 0.126953125, + "rewards/accuracy_reward/std": 0.33324605226516724, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.9267578125, + "rewards/tag_count_reward/std": 0.1889999359846115, + "step": 1802 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1858.0, + "completions/mean_length": 806.29296875, + "completions/mean_terminated_length": 758.4381103515625, + "completions/min_length": 170.0, + "completions/min_terminated_length": 170.0, + "epoch": 0.6155159170436119, + "grad_norm": 1.382660984992981, + "kl": 6.4765625, + "learning_rate": 4.4822361896242734e-07, + "loss": 0.4099, + "num_tokens": 996374256.0, + "reward": 1.85546875, + "reward_std": 0.5291862487792969, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.923828125, + "rewards/tag_count_reward/std": 0.2070399820804596, + "step": 1803 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1995.0, + "completions/mean_length": 767.041015625, + "completions/mean_terminated_length": 725.7197265625, + "completions/min_length": 133.0, + "completions/min_terminated_length": 133.0, + "epoch": 0.6158573013570027, + "grad_norm": 1.1367708444595337, + "kl": 5.271484375, + "learning_rate": 4.477014750303951e-07, + "loss": 0.3426, + "num_tokens": 996849669.0, + "reward": 1.8408203125, + "reward_std": 0.4900355339050293, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.9287109375, + "rewards/tag_count_reward/std": 0.1935756355524063, + "step": 1804 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1948.0, + "completions/mean_length": 764.02734375, + "completions/mean_terminated_length": 727.9317016601562, + "completions/min_length": 145.0, + "completions/min_terminated_length": 145.0, + "epoch": 0.6161986856703935, + "grad_norm": 1.0454238653182983, + "kl": 5.19921875, + "learning_rate": 4.471794762924084e-07, + "loss": 0.3482, + "num_tokens": 997314259.0, + "reward": 1.9091796875, + "reward_std": 0.47368374466896057, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.8828125, + "rewards/format_reward/std": 0.32195815443992615, + "rewards/tag_count_reward/mean": 0.9423828125, + "rewards/tag_count_reward/std": 0.16946613788604736, + "step": 1805 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1908.0, + "completions/mean_length": 706.70703125, + "completions/mean_terminated_length": 688.1148681640625, + "completions/min_length": 170.0, + "completions/min_terminated_length": 170.0, + "epoch": 0.6165400699837843, + "grad_norm": 3.4908785820007324, + "kl": 4.6171875, + "learning_rate": 4.46657623489349e-07, + "loss": 0.317, + "num_tokens": 997757597.0, + "reward": 1.91357421875, + "reward_std": 0.5355505347251892, + "rewards/accuracy_reward/mean": 0.1015625, + "rewards/accuracy_reward/std": 0.30236753821372986, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.94091796875, + "rewards/tag_count_reward/std": 0.17360158264636993, + "step": 1806 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1768.0, + "completions/mean_length": 762.546875, + "completions/mean_terminated_length": 723.75048828125, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "epoch": 0.616881454297175, + "grad_norm": 1.0735392570495605, + "kl": 5.5859375, + "learning_rate": 4.461359173618914e-07, + "loss": 0.3496, + "num_tokens": 998232741.0, + "reward": 1.83740234375, + "reward_std": 0.5108038187026978, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.826171875, + "rewards/format_reward/std": 0.3793322443962097, + "rewards/tag_count_reward/mean": 0.92529296875, + "rewards/tag_count_reward/std": 0.18874886631965637, + "step": 1807 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2022.0, + "completions/mean_length": 814.939453125, + "completions/mean_terminated_length": 764.8150024414062, + "completions/min_length": 194.0, + "completions/min_terminated_length": 194.0, + "epoch": 0.6172228386105658, + "grad_norm": 1.6330175399780273, + "kl": 6.30078125, + "learning_rate": 4.456143586505021e-07, + "loss": 0.3631, + "num_tokens": 998732806.0, + "reward": 1.79150390625, + "reward_std": 0.5776578187942505, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.830078125, + "rewards/format_reward/std": 0.3759314715862274, + "rewards/tag_count_reward/mean": 0.91455078125, + "rewards/tag_count_reward/std": 0.21192020177841187, + "step": 1808 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1989.0, + "completions/mean_length": 761.005859375, + "completions/mean_terminated_length": 719.4898681640625, + "completions/min_length": 127.0, + "completions/min_terminated_length": 127.0, + "epoch": 0.6175642229239566, + "grad_norm": 1.2343679666519165, + "kl": 6.296875, + "learning_rate": 4.450929480954383e-07, + "loss": 0.3686, + "num_tokens": 999194809.0, + "reward": 1.79248046875, + "reward_std": 0.5539897680282593, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.8359375, + "rewards/format_reward/std": 0.37069445848464966, + "rewards/tag_count_reward/mean": 0.91748046875, + "rewards/tag_count_reward/std": 0.20248481631278992, + "step": 1809 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1890.0, + "completions/mean_length": 743.8515625, + "completions/mean_terminated_length": 704.4909057617188, + "completions/min_length": 169.0, + "completions/min_terminated_length": 169.0, + "epoch": 0.6179056072373474, + "grad_norm": 1.5132052898406982, + "kl": 5.5703125, + "learning_rate": 4.445716864367472e-07, + "loss": 0.3359, + "num_tokens": 999653501.0, + "reward": 1.921875, + "reward_std": 0.5488927960395813, + "rewards/accuracy_reward/mean": 0.12109375, + "rewards/accuracy_reward/std": 0.3265552520751953, + "rewards/format_reward/mean": 0.8671875, + "rewards/format_reward/std": 0.33970388770103455, + "rewards/tag_count_reward/mean": 0.93359375, + "rewards/tag_count_reward/std": 0.1869896799325943, + "step": 1810 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2015.0, + "completions/mean_length": 714.630859375, + "completions/mean_terminated_length": 696.1485595703125, + "completions/min_length": 140.0, + "completions/min_terminated_length": 140.0, + "epoch": 0.6182469915507383, + "grad_norm": 2.2754740715026855, + "kl": 4.828125, + "learning_rate": 4.440505744142639e-07, + "loss": 0.2989, + "num_tokens": 1000096352.0, + "reward": 1.88818359375, + "reward_std": 0.4665547311306, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.876953125, + "rewards/format_reward/std": 0.32881227135658264, + "rewards/tag_count_reward/mean": 0.94482421875, + "rewards/tag_count_reward/std": 0.16480305790901184, + "step": 1811 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1983.0, + "completions/mean_length": 770.412109375, + "completions/mean_terminated_length": 731.8530883789062, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "epoch": 0.6185883758641291, + "grad_norm": 1.864904761314392, + "kl": 7.265625, + "learning_rate": 4.4352961276761183e-07, + "loss": 0.4279, + "num_tokens": 1000562099.0, + "reward": 1.8359375, + "reward_std": 0.5089847445487976, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.841796875, + "rewards/format_reward/std": 0.36528825759887695, + "rewards/tag_count_reward/mean": 0.9296875, + "rewards/tag_count_reward/std": 0.1835651993751526, + "step": 1812 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1941.0, + "completions/mean_length": 731.8046875, + "completions/mean_terminated_length": 713.5604248046875, + "completions/min_length": 126.0, + "completions/min_terminated_length": 126.0, + "epoch": 0.6189297601775199, + "grad_norm": 1.3424431085586548, + "kl": 5.5, + "learning_rate": 4.4300880223620063e-07, + "loss": 0.3374, + "num_tokens": 1001009471.0, + "reward": 1.8642578125, + "reward_std": 0.4546358585357666, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.875, + "rewards/format_reward/std": 0.3310423493385315, + "rewards/tag_count_reward/mean": 0.9423828125, + "rewards/tag_count_reward/std": 0.17161767184734344, + "step": 1813 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1961.0, + "completions/mean_length": 764.001953125, + "completions/mean_terminated_length": 733.18603515625, + "completions/min_length": 121.0, + "completions/min_terminated_length": 121.0, + "epoch": 0.6192711444909107, + "grad_norm": 2.8516862392425537, + "kl": 5.8671875, + "learning_rate": 4.424881435592256e-07, + "loss": 0.3957, + "num_tokens": 1001477152.0, + "reward": 1.859375, + "reward_std": 0.47271546721458435, + "rewards/accuracy_reward/mean": 0.03515625, + "rewards/accuracy_reward/std": 0.1843547374010086, + "rewards/format_reward/mean": 0.880859375, + "rewards/format_reward/std": 0.32427072525024414, + "rewards/tag_count_reward/mean": 0.943359375, + "rewards/tag_count_reward/std": 0.1726529598236084, + "step": 1814 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1981.0, + "completions/mean_length": 850.126953125, + "completions/mean_terminated_length": 801.4329223632812, + "completions/min_length": 216.0, + "completions/min_terminated_length": 216.0, + "epoch": 0.6196125288043014, + "grad_norm": 1.2924460172653198, + "kl": 6.515625, + "learning_rate": 4.419676374756668e-07, + "loss": 0.4286, + "num_tokens": 1001991457.0, + "reward": 1.88134765625, + "reward_std": 0.5008226633071899, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.857421875, + "rewards/format_reward/std": 0.3499840497970581, + "rewards/tag_count_reward/mean": 0.92626953125, + "rewards/tag_count_reward/std": 0.19549313187599182, + "step": 1815 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 789.578125, + "completions/mean_terminated_length": 724.9774169921875, + "completions/min_length": 210.0, + "completions/min_terminated_length": 210.0, + "epoch": 0.6199539131176922, + "grad_norm": 1.3212380409240723, + "kl": 7.484375, + "learning_rate": 4.4144728472428725e-07, + "loss": 0.4626, + "num_tokens": 1002470137.0, + "reward": 1.88623046875, + "reward_std": 0.5643240809440613, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.857421875, + "rewards/format_reward/std": 0.3499840497970581, + "rewards/tag_count_reward/mean": 0.92333984375, + "rewards/tag_count_reward/std": 0.20477943122386932, + "step": 1816 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 766.20703125, + "completions/mean_terminated_length": 700.4066162109375, + "completions/min_length": 7.0, + "completions/min_terminated_length": 7.0, + "epoch": 0.620295297431083, + "grad_norm": 1.9985350370407104, + "kl": 8.0390625, + "learning_rate": 4.409270860436325e-07, + "loss": 0.494, + "num_tokens": 1002947075.0, + "reward": 1.79736328125, + "reward_std": 0.5326906442642212, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.841796875, + "rewards/format_reward/std": 0.36528825759887695, + "rewards/tag_count_reward/mean": 0.91455078125, + "rewards/tag_count_reward/std": 0.21192020177841187, + "step": 1817 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2006.0, + "completions/mean_length": 737.626953125, + "completions/mean_terminated_length": 692.624267578125, + "completions/min_length": 149.0, + "completions/min_terminated_length": 149.0, + "epoch": 0.6206366817444738, + "grad_norm": 2.6582906246185303, + "kl": 7.6328125, + "learning_rate": 4.404070421720293e-07, + "loss": 0.4606, + "num_tokens": 1003409060.0, + "reward": 1.85595703125, + "reward_std": 0.5522315502166748, + "rewards/accuracy_reward/mean": 0.12890625, + "rewards/accuracy_reward/std": 0.33542385697364807, + "rewards/format_reward/mean": 0.81640625, + "rewards/format_reward/std": 0.3875311613082886, + "rewards/tag_count_reward/mean": 0.91064453125, + "rewards/tag_count_reward/std": 0.20380185544490814, + "step": 1818 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1700.0, + "completions/mean_length": 777.990234375, + "completions/mean_terminated_length": 750.1057739257812, + "completions/min_length": 161.0, + "completions/min_terminated_length": 161.0, + "epoch": 0.6209780660578647, + "grad_norm": 1.1062575578689575, + "kl": 4.875, + "learning_rate": 4.398871538475849e-07, + "loss": 0.3062, + "num_tokens": 1003889167.0, + "reward": 1.8896484375, + "reward_std": 0.4880656599998474, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.875, + "rewards/format_reward/std": 0.3310423493385315, + "rewards/tag_count_reward/mean": 0.9482421875, + "rewards/tag_count_reward/std": 0.1610473245382309, + "step": 1819 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1984.0, + "completions/mean_length": 829.810546875, + "completions/mean_terminated_length": 759.3367309570312, + "completions/min_length": 186.0, + "completions/min_terminated_length": 186.0, + "epoch": 0.6213194503712555, + "grad_norm": 1.053416132926941, + "kl": 7.3125, + "learning_rate": 4.3936742180818565e-07, + "loss": 0.4901, + "num_tokens": 1004390254.0, + "reward": 1.8740234375, + "reward_std": 0.5669934153556824, + "rewards/accuracy_reward/mean": 0.1015625, + "rewards/accuracy_reward/std": 0.30236753821372986, + "rewards/format_reward/mean": 0.849609375, + "rewards/format_reward/std": 0.35780346393585205, + "rewards/tag_count_reward/mean": 0.9228515625, + "rewards/tag_count_reward/std": 0.20369693636894226, + "step": 1820 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2017.0, + "completions/mean_length": 806.140625, + "completions/mean_terminated_length": 768.659912109375, + "completions/min_length": 107.0, + "completions/min_terminated_length": 107.0, + "epoch": 0.6216608346846463, + "grad_norm": 1.6496962308883667, + "kl": 5.89453125, + "learning_rate": 4.3884784679149613e-07, + "loss": 0.3358, + "num_tokens": 1004888854.0, + "reward": 1.81494140625, + "reward_std": 0.5355316996574402, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.91650390625, + "rewards/tag_count_reward/std": 0.2032901644706726, + "step": 1821 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1969.0, + "completions/mean_length": 825.744140625, + "completions/mean_terminated_length": 778.638916015625, + "completions/min_length": 187.0, + "completions/min_terminated_length": 187.0, + "epoch": 0.6220022189980371, + "grad_norm": 1.900420069694519, + "kl": 6.1875, + "learning_rate": 4.383284295349583e-07, + "loss": 0.3893, + "num_tokens": 1005392083.0, + "reward": 1.81201171875, + "reward_std": 0.5551861524581909, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.8359375, + "rewards/format_reward/std": 0.37069445848464966, + "rewards/tag_count_reward/mean": 0.92138671875, + "rewards/tag_count_reward/std": 0.1948370635509491, + "step": 1822 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 800.244140625, + "completions/mean_terminated_length": 757.3919677734375, + "completions/min_length": 189.0, + "completions/min_terminated_length": 189.0, + "epoch": 0.6223436033114278, + "grad_norm": 1.5868628025054932, + "kl": 6.7421875, + "learning_rate": 4.378091707757896e-07, + "loss": 0.4093, + "num_tokens": 1005881280.0, + "reward": 1.8447265625, + "reward_std": 0.6214015483856201, + "rewards/accuracy_reward/mean": 0.1015625, + "rewards/accuracy_reward/std": 0.30236753821372986, + "rewards/format_reward/mean": 0.833984375, + "rewards/format_reward/std": 0.3724585771560669, + "rewards/tag_count_reward/mean": 0.9091796875, + "rewards/tag_count_reward/std": 0.21457123756408691, + "step": 1823 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2041.0, + "completions/mean_length": 834.66796875, + "completions/mean_terminated_length": 767.1217041015625, + "completions/min_length": 143.0, + "completions/min_terminated_length": 143.0, + "epoch": 0.6226849876248186, + "grad_norm": 1.5598671436309814, + "kl": 6.4296875, + "learning_rate": 4.372900712509831e-07, + "loss": 0.3905, + "num_tokens": 1006383382.0, + "reward": 1.81005859375, + "reward_std": 0.5211690664291382, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.91552734375, + "rewards/tag_count_reward/std": 0.20882701873779297, + "step": 1824 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 746.78515625, + "completions/mean_terminated_length": 702.0969848632812, + "completions/min_length": 17.0, + "completions/min_terminated_length": 17.0, + "epoch": 0.6230263719382094, + "grad_norm": 1.1952095031738281, + "kl": 6.10546875, + "learning_rate": 4.367711316973054e-07, + "loss": 0.3898, + "num_tokens": 1006836584.0, + "reward": 1.86181640625, + "reward_std": 0.4903917908668518, + "rewards/accuracy_reward/mean": 0.07258064299821854, + "rewards/accuracy_reward/std": 0.25970885157585144, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.93017578125, + "rewards/tag_count_reward/std": 0.18737702071666718, + "step": 1825 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1987.0, + "completions/mean_length": 799.009765625, + "completions/mean_terminated_length": 758.7197265625, + "completions/min_length": 146.0, + "completions/min_terminated_length": 146.0, + "epoch": 0.6233677562516002, + "grad_norm": 1.1136211156845093, + "kl": 6.1484375, + "learning_rate": 4.3625235285129634e-07, + "loss": 0.3822, + "num_tokens": 1007322141.0, + "reward": 1.91552734375, + "reward_std": 0.5885144472122192, + "rewards/accuracy_reward/mean": 0.138671875, + "rewards/accuracy_reward/std": 0.34594178199768066, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.92529296875, + "rewards/tag_count_reward/std": 0.19574713706970215, + "step": 1826 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1987.0, + "completions/mean_length": 823.900390625, + "completions/mean_terminated_length": 758.41357421875, + "completions/min_length": 44.0, + "completions/min_terminated_length": 44.0, + "epoch": 0.623709140564991, + "grad_norm": 1.2269648313522339, + "kl": 7.3359375, + "learning_rate": 4.3573373544926786e-07, + "loss": 0.4647, + "num_tokens": 1007818730.0, + "reward": 1.80322265625, + "reward_std": 0.6270456314086914, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.810546875, + "rewards/format_reward/std": 0.3922513723373413, + "rewards/tag_count_reward/mean": 0.90673828125, + "rewards/tag_count_reward/std": 0.221117302775383, + "step": 1827 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 821.3515625, + "completions/mean_terminated_length": 771.48779296875, + "completions/min_length": 125.0, + "completions/min_terminated_length": 125.0, + "epoch": 0.6240505248783819, + "grad_norm": 1.0753824710845947, + "kl": 7.8203125, + "learning_rate": 4.352152802273024e-07, + "loss": 0.4759, + "num_tokens": 1008320846.0, + "reward": 1.8017578125, + "reward_std": 0.5702003240585327, + "rewards/accuracy_reward/mean": 0.0786290317773819, + "rewards/accuracy_reward/std": 0.26943066716194153, + "rewards/format_reward/mean": 0.8203125, + "rewards/format_reward/std": 0.38430243730545044, + "rewards/tag_count_reward/mean": 0.9052734375, + "rewards/tag_count_reward/std": 0.21572597324848175, + "step": 1828 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 831.482421875, + "completions/mean_terminated_length": 789.7030639648438, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "epoch": 0.6243919091917727, + "grad_norm": 1.256855845451355, + "kl": 5.09375, + "learning_rate": 4.3469698792125196e-07, + "loss": 0.3163, + "num_tokens": 1008822181.0, + "reward": 1.82080078125, + "reward_std": 0.48862963914871216, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.92822265625, + "rewards/tag_count_reward/std": 0.18399609625339508, + "step": 1829 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2028.0, + "completions/mean_length": 788.90625, + "completions/mean_terminated_length": 758.6880493164062, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "epoch": 0.6247332935051635, + "grad_norm": 2.1441006660461426, + "kl": 6.0390625, + "learning_rate": 4.341788592667381e-07, + "loss": 0.3577, + "num_tokens": 1009302837.0, + "reward": 1.83056640625, + "reward_std": 0.5533666610717773, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.8359375, + "rewards/format_reward/std": 0.37069445848464966, + "rewards/tag_count_reward/mean": 0.91845703125, + "rewards/tag_count_reward/std": 0.19362683594226837, + "step": 1830 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1890.0, + "completions/mean_length": 750.34765625, + "completions/mean_terminated_length": 716.5410766601562, + "completions/min_length": 192.0, + "completions/min_terminated_length": 192.0, + "epoch": 0.6250746778185542, + "grad_norm": 0.9880239367485046, + "kl": 4.8359375, + "learning_rate": 4.336608949991496e-07, + "loss": 0.2965, + "num_tokens": 1009770151.0, + "reward": 1.9775390625, + "reward_std": 0.5535204410552979, + "rewards/accuracy_reward/mean": 0.16796875, + "rewards/accuracy_reward/std": 0.374204158782959, + "rewards/format_reward/mean": 0.873046875, + "rewards/format_reward/std": 0.33324605226516724, + "rewards/tag_count_reward/mean": 0.9365234375, + "rewards/tag_count_reward/std": 0.17934982478618622, + "step": 1831 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2014.0, + "completions/mean_length": 854.609375, + "completions/mean_terminated_length": 793.3470458984375, + "completions/min_length": 150.0, + "completions/min_terminated_length": 150.0, + "epoch": 0.625416062131945, + "grad_norm": 1.020838737487793, + "kl": 6.22265625, + "learning_rate": 4.3314309585364185e-07, + "loss": 0.3637, + "num_tokens": 1010290687.0, + "reward": 1.85107421875, + "reward_std": 0.5193954706192017, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.86328125, + "rewards/format_reward/std": 0.3438861668109894, + "rewards/tag_count_reward/mean": 0.92138671875, + "rewards/tag_count_reward/std": 0.20463472604751587, + "step": 1832 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 770.44140625, + "completions/mean_terminated_length": 755.2925415039062, + "completions/min_length": 190.0, + "completions/min_terminated_length": 190.0, + "epoch": 0.6257574464453358, + "grad_norm": 1.6976969242095947, + "kl": 4.18359375, + "learning_rate": 4.3262546256513613e-07, + "loss": 0.2874, + "num_tokens": 1010764705.0, + "reward": 1.9326171875, + "reward_std": 0.48232823610305786, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.8984375, + "rewards/format_reward/std": 0.30236753821372986, + "rewards/tag_count_reward/mean": 0.9482421875, + "rewards/tag_count_reward/std": 0.16330981254577637, + "step": 1833 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1993.0, + "completions/mean_length": 862.01171875, + "completions/mean_terminated_length": 816.3042602539062, + "completions/min_length": 134.0, + "completions/min_terminated_length": 134.0, + "epoch": 0.6260988307587266, + "grad_norm": 1.7305068969726562, + "kl": 5.62890625, + "learning_rate": 4.3210799586831825e-07, + "loss": 0.3433, + "num_tokens": 1011283879.0, + "reward": 1.79931640625, + "reward_std": 0.544184148311615, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.833984375, + "rewards/format_reward/std": 0.3724585771560669, + "rewards/tag_count_reward/mean": 0.91650390625, + "rewards/tag_count_reward/std": 0.1946849673986435, + "step": 1834 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1917.0, + "completions/mean_length": 850.76953125, + "completions/mean_terminated_length": 797.0162963867188, + "completions/min_length": 174.0, + "completions/min_terminated_length": 174.0, + "epoch": 0.6264402150721174, + "grad_norm": 1.5599923133850098, + "kl": 6.9375, + "learning_rate": 4.3159069649763747e-07, + "loss": 0.4452, + "num_tokens": 1011796913.0, + "reward": 1.828125, + "reward_std": 0.5796926021575928, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.916015625, + "rewards/tag_count_reward/std": 0.2087314873933792, + "step": 1835 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2019.0, + "completions/mean_length": 829.044921875, + "completions/mean_terminated_length": 799.7900390625, + "completions/min_length": 178.0, + "completions/min_terminated_length": 178.0, + "epoch": 0.6267815993855083, + "grad_norm": 0.9597150683403015, + "kl": 6.125, + "learning_rate": 4.3107356518730564e-07, + "loss": 0.3887, + "num_tokens": 1012307032.0, + "reward": 1.84912109375, + "reward_std": 0.528598427772522, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.93310546875, + "rewards/tag_count_reward/std": 0.1749831587076187, + "step": 1836 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1897.0, + "completions/mean_length": 825.412109375, + "completions/mean_terminated_length": 780.8643798828125, + "completions/min_length": 75.0, + "completions/min_terminated_length": 75.0, + "epoch": 0.6271229836988991, + "grad_norm": 1.6486892700195312, + "kl": 6.0703125, + "learning_rate": 4.30556602671296e-07, + "loss": 0.3711, + "num_tokens": 1012802251.0, + "reward": 1.85693359375, + "reward_std": 0.5106162428855896, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.8671875, + "rewards/format_reward/std": 0.33970388770103455, + "rewards/tag_count_reward/mean": 0.92724609375, + "rewards/tag_count_reward/std": 0.19079817831516266, + "step": 1837 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 832.314453125, + "completions/mean_terminated_length": 772.526611328125, + "completions/min_length": 82.0, + "completions/min_terminated_length": 82.0, + "epoch": 0.6274643680122899, + "grad_norm": 2.0939388275146484, + "kl": 7.7578125, + "learning_rate": 4.300398096833424e-07, + "loss": 0.4827, + "num_tokens": 1013303324.0, + "reward": 1.84375, + "reward_std": 0.5772364735603333, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.916015625, + "rewards/tag_count_reward/std": 0.20931662619113922, + "step": 1838 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 775.90234375, + "completions/mean_terminated_length": 745.3720092773438, + "completions/min_length": 156.0, + "completions/min_terminated_length": 156.0, + "epoch": 0.6278057523256806, + "grad_norm": 2.218848943710327, + "kl": 6.140625, + "learning_rate": 4.2952318695693803e-07, + "loss": 0.3746, + "num_tokens": 1013777194.0, + "reward": 1.84228515625, + "reward_std": 0.561437726020813, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.841796875, + "rewards/format_reward/std": 0.36528825759887695, + "rewards/tag_count_reward/mean": 0.92431640625, + "rewards/tag_count_reward/std": 0.1902967095375061, + "step": 1839 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 789.443359375, + "completions/mean_terminated_length": 738.282470703125, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.6281471366390714, + "grad_norm": 4.587276935577393, + "kl": 8.5234375, + "learning_rate": 4.290067352253344e-07, + "loss": 0.4924, + "num_tokens": 1014250861.0, + "reward": 1.7724609375, + "reward_std": 0.5534372329711914, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.822265625, + "rewards/format_reward/std": 0.3826628625392914, + "rewards/tag_count_reward/mean": 0.9091796875, + "rewards/tag_count_reward/std": 0.20996157824993134, + "step": 1840 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2033.0, + "completions/mean_length": 776.703125, + "completions/mean_terminated_length": 740.9638061523438, + "completions/min_length": 178.0, + "completions/min_terminated_length": 178.0, + "epoch": 0.6284885209524622, + "grad_norm": 3.668210029602051, + "kl": 7.21875, + "learning_rate": 4.2849045522153994e-07, + "loss": 0.4118, + "num_tokens": 1014729573.0, + "reward": 1.7890625, + "reward_std": 0.5655283331871033, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.8125, + "rewards/format_reward/std": 0.39069411158561707, + "rewards/tag_count_reward/mean": 0.9140625, + "rewards/tag_count_reward/std": 0.1907537281513214, + "step": 1841 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 837.736328125, + "completions/mean_terminated_length": 788.53857421875, + "completions/min_length": 144.0, + "completions/min_terminated_length": 144.0, + "epoch": 0.628829905265853, + "grad_norm": 5.375893592834473, + "kl": 8.25, + "learning_rate": 4.2797434767831985e-07, + "loss": 0.453, + "num_tokens": 1015235342.0, + "reward": 1.7314453125, + "reward_std": 0.6241041421890259, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.76953125, + "rewards/format_reward/std": 0.42154473066329956, + "rewards/tag_count_reward/mean": 0.8935546875, + "rewards/tag_count_reward/std": 0.21421466767787933, + "step": 1842 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2003.0, + "completions/mean_length": 783.0625, + "completions/mean_terminated_length": 736.9716796875, + "completions/min_length": 189.0, + "completions/min_terminated_length": 189.0, + "epoch": 0.6291712895792438, + "grad_norm": 1.8987025022506714, + "kl": 6.8359375, + "learning_rate": 4.274584133281944e-07, + "loss": 0.418, + "num_tokens": 1015703998.0, + "reward": 1.8291015625, + "reward_std": 0.5494622588157654, + "rewards/accuracy_reward/mean": 0.05645161122083664, + "rewards/accuracy_reward/std": 0.23102475702762604, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.9228515625, + "rewards/tag_count_reward/std": 0.18873685598373413, + "step": 1843 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1935.0, + "completions/mean_length": 771.59375, + "completions/mean_terminated_length": 719.707275390625, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "epoch": 0.6295126738926347, + "grad_norm": 1.2763030529022217, + "kl": 6.5234375, + "learning_rate": 4.269426529034382e-07, + "loss": 0.4324, + "num_tokens": 1016175038.0, + "reward": 1.85546875, + "reward_std": 0.5567238330841064, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.857421875, + "rewards/format_reward/std": 0.3499840497970581, + "rewards/tag_count_reward/mean": 0.921875, + "rewards/tag_count_reward/std": 0.19597215950489044, + "step": 1844 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1772.0, + "completions/mean_length": 727.19921875, + "completions/mean_terminated_length": 700.8884887695312, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "epoch": 0.6298540582060255, + "grad_norm": 3.3444409370422363, + "kl": 4.765625, + "learning_rate": 4.264270671360787e-07, + "loss": 0.2975, + "num_tokens": 1016619300.0, + "reward": 1.84521484375, + "reward_std": 0.5264946818351746, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.849609375, + "rewards/format_reward/std": 0.35780346393585205, + "rewards/tag_count_reward/mean": 0.92138671875, + "rewards/tag_count_reward/std": 0.1884550154209137, + "step": 1845 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1951.0, + "completions/mean_length": 775.34375, + "completions/mean_terminated_length": 739.5662231445312, + "completions/min_length": 101.0, + "completions/min_terminated_length": 101.0, + "epoch": 0.6301954425194163, + "grad_norm": 1.9668244123458862, + "kl": 4.640625, + "learning_rate": 4.2591165675789555e-07, + "loss": 0.3106, + "num_tokens": 1017097236.0, + "reward": 1.91455078125, + "reward_std": 0.5258707404136658, + "rewards/accuracy_reward/mean": 0.11290322244167328, + "rewards/accuracy_reward/std": 0.3167939782142639, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.93603515625, + "rewards/tag_count_reward/std": 0.17468255758285522, + "step": 1846 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1959.0, + "completions/mean_length": 769.189453125, + "completions/mean_terminated_length": 748.8909301757812, + "completions/min_length": 213.0, + "completions/min_terminated_length": 213.0, + "epoch": 0.630536826832807, + "grad_norm": 2.781959295272827, + "kl": 4.33203125, + "learning_rate": 4.2539642250041973e-07, + "loss": 0.2909, + "num_tokens": 1017562949.0, + "reward": 1.9033203125, + "reward_std": 0.4896920323371887, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.89453125, + "rewards/format_reward/std": 0.3074568510055542, + "rewards/tag_count_reward/mean": 0.9462890625, + "rewards/tag_count_reward/std": 0.16342678666114807, + "step": 1847 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1936.0, + "completions/mean_length": 783.32421875, + "completions/mean_terminated_length": 734.5841674804688, + "completions/min_length": 109.0, + "completions/min_terminated_length": 109.0, + "epoch": 0.6308782111461978, + "grad_norm": 2.7658309936523438, + "kl": 5.9921875, + "learning_rate": 4.2488136509493165e-07, + "loss": 0.4025, + "num_tokens": 1018037451.0, + "reward": 1.82275390625, + "reward_std": 0.5239130258560181, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.857421875, + "rewards/format_reward/std": 0.3499840497970581, + "rewards/tag_count_reward/mean": 0.92041015625, + "rewards/tag_count_reward/std": 0.20123985409736633, + "step": 1848 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2009.0, + "completions/mean_length": 759.548828125, + "completions/mean_terminated_length": 715.2990112304688, + "completions/min_length": 169.0, + "completions/min_terminated_length": 169.0, + "epoch": 0.6312195954595886, + "grad_norm": 2.1002485752105713, + "kl": 5.515625, + "learning_rate": 4.243664852724614e-07, + "loss": 0.3747, + "num_tokens": 1018505444.0, + "reward": 1.8642578125, + "reward_std": 0.5158007144927979, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.9345703125, + "rewards/tag_count_reward/std": 0.17795921862125397, + "step": 1849 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1987.0, + "completions/mean_length": 802.892578125, + "completions/mean_terminated_length": 746.98974609375, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "epoch": 0.6315609797729794, + "grad_norm": 1.6585819721221924, + "kl": 6.41015625, + "learning_rate": 4.238517837637866e-07, + "loss": 0.3989, + "num_tokens": 1019012141.0, + "reward": 1.87451171875, + "reward_std": 0.501410961151123, + "rewards/accuracy_reward/mean": 0.08064515888690948, + "rewards/accuracy_reward/std": 0.2725643217563629, + "rewards/format_reward/mean": 0.8671875, + "rewards/format_reward/std": 0.33970388770103455, + "rewards/tag_count_reward/mean": 0.92919921875, + "rewards/tag_count_reward/std": 0.18960754573345184, + "step": 1850 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1970.0, + "completions/mean_length": 792.677734375, + "completions/mean_terminated_length": 752.1834716796875, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "epoch": 0.6319023640863702, + "grad_norm": 1.6555026769638062, + "kl": 6.640625, + "learning_rate": 4.233372612994319e-07, + "loss": 0.4092, + "num_tokens": 1019495976.0, + "reward": 1.85302734375, + "reward_std": 0.5476795434951782, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.91552734375, + "rewards/tag_count_reward/std": 0.20228178799152374, + "step": 1851 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2018.0, + "completions/mean_length": 859.576171875, + "completions/mean_terminated_length": 816.2733154296875, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "epoch": 0.632243748399761, + "grad_norm": 1.5367271900177002, + "kl": 6.8203125, + "learning_rate": 4.228229186096679e-07, + "loss": 0.3944, + "num_tokens": 1020011855.0, + "reward": 1.79443359375, + "reward_std": 0.5380985140800476, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.90966796875, + "rewards/tag_count_reward/std": 0.21844784915447235, + "step": 1852 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1990.0, + "completions/mean_length": 805.333984375, + "completions/mean_terminated_length": 752.1853637695312, + "completions/min_length": 119.0, + "completions/min_terminated_length": 119.0, + "epoch": 0.6325851327131519, + "grad_norm": 1.0009610652923584, + "kl": 6.8828125, + "learning_rate": 4.223087564245099e-07, + "loss": 0.4441, + "num_tokens": 1020514202.0, + "reward": 1.79931640625, + "reward_std": 0.5320873856544495, + "rewards/accuracy_reward/mean": 0.03125, + "rewards/accuracy_reward/std": 0.17416280508041382, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.92041015625, + "rewards/tag_count_reward/std": 0.20425614714622498, + "step": 1853 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 892.333984375, + "completions/mean_terminated_length": 850.2247314453125, + "completions/min_length": 156.0, + "completions/min_terminated_length": 156.0, + "epoch": 0.6329265170265427, + "grad_norm": 1.004145622253418, + "kl": 5.7109375, + "learning_rate": 4.2179477547371713e-07, + "loss": 0.3546, + "num_tokens": 1021045461.0, + "reward": 1.82275390625, + "reward_std": 0.5507851839065552, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.92041015625, + "rewards/tag_count_reward/std": 0.19940820336341858, + "step": 1854 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2003.0, + "completions/mean_length": 768.025390625, + "completions/mean_terminated_length": 734.6793823242188, + "completions/min_length": 170.0, + "completions/min_terminated_length": 170.0, + "epoch": 0.6332679013399334, + "grad_norm": 0.9820337295532227, + "kl": 6.5859375, + "learning_rate": 4.212809764867915e-07, + "loss": 0.4173, + "num_tokens": 1021515906.0, + "reward": 1.83056640625, + "reward_std": 0.5448594689369202, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.92041015625, + "rewards/tag_count_reward/std": 0.19879388809204102, + "step": 1855 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1992.0, + "completions/mean_length": 792.013671875, + "completions/mean_terminated_length": 746.2490234375, + "completions/min_length": 176.0, + "completions/min_terminated_length": 176.0, + "epoch": 0.6336092856533242, + "grad_norm": 1.5900251865386963, + "kl": 7.6484375, + "learning_rate": 4.2076736019297674e-07, + "loss": 0.4744, + "num_tokens": 1021996073.0, + "reward": 1.8427734375, + "reward_std": 0.5668472647666931, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.849609375, + "rewards/format_reward/std": 0.35780346393585205, + "rewards/tag_count_reward/mean": 0.9228515625, + "rewards/tag_count_reward/std": 0.19635941088199615, + "step": 1856 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1994.0, + "completions/mean_length": 750.109375, + "completions/mean_terminated_length": 697.3495483398438, + "completions/min_length": 110.0, + "completions/min_terminated_length": 110.0, + "epoch": 0.633950669966715, + "grad_norm": 1.420174479484558, + "kl": 5.787109375, + "learning_rate": 4.202539273212572e-07, + "loss": 0.3654, + "num_tokens": 1022461153.0, + "reward": 1.83740234375, + "reward_std": 0.5509225130081177, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.92333984375, + "rewards/tag_count_reward/std": 0.2005549520254135, + "step": 1857 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2023.0, + "completions/mean_length": 773.87890625, + "completions/mean_terminated_length": 722.0853271484375, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "epoch": 0.6342920542801058, + "grad_norm": 1.0452171564102173, + "kl": 6.46875, + "learning_rate": 4.197406786003569e-07, + "loss": 0.4259, + "num_tokens": 1022933731.0, + "reward": 1.83984375, + "reward_std": 0.5241327881813049, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.91796875, + "rewards/tag_count_reward/std": 0.2059757262468338, + "step": 1858 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 769.501953125, + "completions/mean_terminated_length": 728.2600708007812, + "completions/min_length": 128.0, + "completions/min_terminated_length": 128.0, + "epoch": 0.6346334385934966, + "grad_norm": 2.23154878616333, + "kl": 6.8671875, + "learning_rate": 4.192276147587387e-07, + "loss": 0.4102, + "num_tokens": 1023403076.0, + "reward": 1.86181640625, + "reward_std": 0.5138826370239258, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.92431640625, + "rewards/tag_count_reward/std": 0.1953708976507187, + "step": 1859 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1982.0, + "completions/mean_length": 810.333984375, + "completions/mean_terminated_length": 762.6348876953125, + "completions/min_length": 195.0, + "completions/min_terminated_length": 195.0, + "epoch": 0.6349748229068874, + "grad_norm": 0.7847080230712891, + "kl": 7.265625, + "learning_rate": 4.1871473652460265e-07, + "loss": 0.465, + "num_tokens": 1023899343.0, + "reward": 1.83837890625, + "reward_std": 0.5229803323745728, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.92626953125, + "rewards/tag_count_reward/std": 0.19297432899475098, + "step": 1860 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1950.0, + "completions/mean_length": 822.53515625, + "completions/mean_terminated_length": 772.719482421875, + "completions/min_length": 107.0, + "completions/min_terminated_length": 107.0, + "epoch": 0.6353162072202783, + "grad_norm": 0.9601285457611084, + "kl": 7.3125, + "learning_rate": 4.182020446258857e-07, + "loss": 0.4548, + "num_tokens": 1024408337.0, + "reward": 1.798828125, + "reward_std": 0.527459979057312, + "rewards/accuracy_reward/mean": 0.03125, + "rewards/accuracy_reward/std": 0.17416280508041382, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.921875, + "rewards/tag_count_reward/std": 0.19534705579280853, + "step": 1861 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1833.0, + "completions/mean_length": 821.248046875, + "completions/mean_terminated_length": 784.2233276367188, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "epoch": 0.6356575915336691, + "grad_norm": 1.136073350906372, + "kl": 7.125, + "learning_rate": 4.1768953979026024e-07, + "loss": 0.3985, + "num_tokens": 1024903952.0, + "reward": 1.841796875, + "reward_std": 0.5617328882217407, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.92578125, + "rewards/tag_count_reward/std": 0.19183269143104553, + "step": 1862 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2000.0, + "completions/mean_length": 766.908203125, + "completions/mean_terminated_length": 736.1620483398438, + "completions/min_length": 70.0, + "completions/min_terminated_length": 70.0, + "epoch": 0.6359989758470598, + "grad_norm": 1.2205417156219482, + "kl": 6.1796875, + "learning_rate": 4.171772227451331e-07, + "loss": 0.3771, + "num_tokens": 1025373601.0, + "reward": 1.90625, + "reward_std": 0.5341774225234985, + "rewards/accuracy_reward/mean": 0.103515625, + "rewards/accuracy_reward/std": 0.30492907762527466, + "rewards/format_reward/mean": 0.873046875, + "rewards/format_reward/std": 0.33324605226516724, + "rewards/tag_count_reward/mean": 0.9296875, + "rewards/tag_count_reward/std": 0.1933014690876007, + "step": 1863 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1950.0, + "completions/mean_length": 775.53515625, + "completions/mean_terminated_length": 726.4949340820312, + "completions/min_length": 99.0, + "completions/min_terminated_length": 99.0, + "epoch": 0.6363403601604506, + "grad_norm": 1.4155833721160889, + "kl": 5.9140625, + "learning_rate": 4.166650942176447e-07, + "loss": 0.3838, + "num_tokens": 1025847987.0, + "reward": 1.8427734375, + "reward_std": 0.5575990676879883, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.9189453125, + "rewards/tag_count_reward/std": 0.2015654444694519, + "step": 1864 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 809.529296875, + "completions/mean_terminated_length": 745.9528198242188, + "completions/min_length": 171.0, + "completions/min_terminated_length": 171.0, + "epoch": 0.6366817444738414, + "grad_norm": 0.8453729152679443, + "kl": 7.4453125, + "learning_rate": 4.1615315493466797e-07, + "loss": 0.5078, + "num_tokens": 1026348754.0, + "reward": 1.82275390625, + "reward_std": 0.5655902028083801, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.83203125, + "rewards/format_reward/std": 0.374204158782959, + "rewards/tag_count_reward/mean": 0.91455078125, + "rewards/tag_count_reward/std": 0.20901454985141754, + "step": 1865 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1835.0, + "completions/mean_length": 736.716796875, + "completions/mean_terminated_length": 694.4172973632812, + "completions/min_length": 148.0, + "completions/min_terminated_length": 148.0, + "epoch": 0.6370231287872322, + "grad_norm": 0.8038697242736816, + "kl": 7.4375, + "learning_rate": 4.156414056228065e-07, + "loss": 0.4755, + "num_tokens": 1026800657.0, + "reward": 1.83056640625, + "reward_std": 0.5810835957527161, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.833984375, + "rewards/format_reward/std": 0.3724585771560669, + "rewards/tag_count_reward/mean": 0.92626953125, + "rewards/tag_count_reward/std": 0.1923394650220871, + "step": 1866 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1982.0, + "completions/mean_length": 731.671875, + "completions/mean_terminated_length": 661.2510375976562, + "completions/min_length": 117.0, + "completions/min_terminated_length": 117.0, + "epoch": 0.637364513100623, + "grad_norm": 1.3973826169967651, + "kl": 7.515625, + "learning_rate": 4.151298470083954e-07, + "loss": 0.4895, + "num_tokens": 1027251289.0, + "reward": 1.85205078125, + "reward_std": 0.5546400547027588, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.849609375, + "rewards/format_reward/std": 0.35780346393585205, + "rewards/tag_count_reward/mean": 0.92236328125, + "rewards/tag_count_reward/std": 0.2020028978586197, + "step": 1867 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1972.0, + "completions/mean_length": 783.55859375, + "completions/mean_terminated_length": 742.7701416015625, + "completions/min_length": 189.0, + "completions/min_terminated_length": 189.0, + "epoch": 0.6377058974140138, + "grad_norm": 1.1674144268035889, + "kl": 6.3125, + "learning_rate": 4.146184798174983e-07, + "loss": 0.3714, + "num_tokens": 1027726983.0, + "reward": 1.88427734375, + "reward_std": 0.581419825553894, + "rewards/accuracy_reward/mean": 0.11328125, + "rewards/accuracy_reward/std": 0.3172462284564972, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.92333984375, + "rewards/tag_count_reward/std": 0.1974821388721466, + "step": 1868 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2012.0, + "completions/mean_length": 762.9296875, + "completions/mean_terminated_length": 729.450927734375, + "completions/min_length": 128.0, + "completions/min_terminated_length": 128.0, + "epoch": 0.6380472817274047, + "grad_norm": 1.9474679231643677, + "kl": 5.96875, + "learning_rate": 4.141073047759076e-07, + "loss": 0.3926, + "num_tokens": 1028195667.0, + "reward": 1.91845703125, + "reward_std": 0.5201526284217834, + "rewards/accuracy_reward/mean": 0.103515625, + "rewards/accuracy_reward/std": 0.30492907762527466, + "rewards/format_reward/mean": 0.873046875, + "rewards/format_reward/std": 0.33324605226516724, + "rewards/tag_count_reward/mean": 0.94189453125, + "rewards/tag_count_reward/std": 0.17180897295475006, + "step": 1869 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 727.17578125, + "completions/mean_terminated_length": 706.2103881835938, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "epoch": 0.6383886660407955, + "grad_norm": 1.4593170881271362, + "kl": 3.56640625, + "learning_rate": 4.135963226091426e-07, + "loss": 0.2256, + "num_tokens": 1028659709.0, + "reward": 1.9609375, + "reward_std": 0.43016159534454346, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.91015625, + "rewards/format_reward/std": 0.2862374484539032, + "rewards/tag_count_reward/mean": 0.9609375, + "rewards/tag_count_reward/std": 0.14141270518302917, + "step": 1870 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2008.0, + "completions/mean_length": 722.958984375, + "completions/mean_terminated_length": 685.7088012695312, + "completions/min_length": 140.0, + "completions/min_terminated_length": 140.0, + "epoch": 0.6387300503541862, + "grad_norm": 1.3785734176635742, + "kl": 4.734375, + "learning_rate": 4.1308553404244927e-07, + "loss": 0.2784, + "num_tokens": 1029110824.0, + "reward": 1.97216796875, + "reward_std": 0.4718218445777893, + "rewards/accuracy_reward/mean": 0.125, + "rewards/accuracy_reward/std": 0.3310423493385315, + "rewards/format_reward/mean": 0.8984375, + "rewards/format_reward/std": 0.30236753821372986, + "rewards/tag_count_reward/mean": 0.94873046875, + "rewards/tag_count_reward/std": 0.1638377457857132, + "step": 1871 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1963.0, + "completions/mean_length": 746.15625, + "completions/mean_terminated_length": 725.4921264648438, + "completions/min_length": 166.0, + "completions/min_terminated_length": 166.0, + "epoch": 0.639071434667577, + "grad_norm": 1.8401607275009155, + "kl": 3.46875, + "learning_rate": 4.1257493980079825e-07, + "loss": 0.2358, + "num_tokens": 1029567480.0, + "reward": 1.96435546875, + "reward_std": 0.3924785256385803, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.919921875, + "rewards/format_reward/std": 0.271679550409317, + "rewards/tag_count_reward/mean": 0.95849609375, + "rewards/tag_count_reward/std": 0.14201197028160095, + "step": 1872 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1972.0, + "completions/mean_length": 779.810546875, + "completions/mean_terminated_length": 749.3740234375, + "completions/min_length": 181.0, + "completions/min_terminated_length": 181.0, + "epoch": 0.6394128189809678, + "grad_norm": 1.39500892162323, + "kl": 4.56640625, + "learning_rate": 4.120645406088846e-07, + "loss": 0.3221, + "num_tokens": 1030036919.0, + "reward": 1.970703125, + "reward_std": 0.4239633083343506, + "rewards/accuracy_reward/mean": 0.087890625, + "rewards/accuracy_reward/std": 0.2834126651287079, + "rewards/format_reward/mean": 0.921875, + "rewards/format_reward/std": 0.26863065361976624, + "rewards/tag_count_reward/mean": 0.9609375, + "rewards/tag_count_reward/std": 0.14141270518302917, + "step": 1873 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1972.0, + "completions/mean_length": 758.751953125, + "completions/mean_terminated_length": 733.0697631835938, + "completions/min_length": 206.0, + "completions/min_terminated_length": 206.0, + "epoch": 0.6397542032943586, + "grad_norm": 1.3300796747207642, + "kl": 3.9140625, + "learning_rate": 4.1155433719112696e-07, + "loss": 0.2804, + "num_tokens": 1030504920.0, + "reward": 1.87744140625, + "reward_std": 0.3875640630722046, + "rewards/accuracy_reward/mean": 0.04032257944345474, + "rewards/accuracy_reward/std": 0.19691328704357147, + "rewards/format_reward/mean": 0.890625, + "rewards/format_reward/std": 0.31241437792778015, + "rewards/tag_count_reward/mean": 0.94775390625, + "rewards/tag_count_reward/std": 0.15820616483688354, + "step": 1874 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2008.0, + "completions/mean_length": 756.87109375, + "completions/mean_terminated_length": 731.1514282226562, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "epoch": 0.6400955876077494, + "grad_norm": 1.3403873443603516, + "kl": 6.0546875, + "learning_rate": 4.1104433027166564e-07, + "loss": 0.3892, + "num_tokens": 1030978454.0, + "reward": 1.8720703125, + "reward_std": 0.4538940191268921, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.884765625, + "rewards/format_reward/std": 0.3196168541908264, + "rewards/tag_count_reward/mean": 0.9443359375, + "rewards/tag_count_reward/std": 0.16574904322624207, + "step": 1875 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1891.0, + "completions/mean_length": 758.2578125, + "completions/mean_terminated_length": 713.9636840820312, + "completions/min_length": 210.0, + "completions/min_terminated_length": 210.0, + "epoch": 0.6404369719211402, + "grad_norm": 1.9409515857696533, + "kl": 7.125, + "learning_rate": 4.1053452057436213e-07, + "loss": 0.4875, + "num_tokens": 1031437210.0, + "reward": 1.84326171875, + "reward_std": 0.5266550183296204, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.93115234375, + "rewards/tag_count_reward/std": 0.18445254862308502, + "step": 1876 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2013.0, + "completions/mean_length": 796.19140625, + "completions/mean_terminated_length": 755.8104858398438, + "completions/min_length": 125.0, + "completions/min_terminated_length": 125.0, + "epoch": 0.640778356234531, + "grad_norm": 2.3897781372070312, + "kl": 8.1328125, + "learning_rate": 4.1002490882279804e-07, + "loss": 0.5263, + "num_tokens": 1031921308.0, + "reward": 1.8662109375, + "reward_std": 0.48998963832855225, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.87890625, + "rewards/format_reward/std": 0.3265552520751953, + "rewards/tag_count_reward/mean": 0.9384765625, + "rewards/tag_count_reward/std": 0.17729215323925018, + "step": 1877 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2003.0, + "completions/mean_length": 710.53515625, + "completions/mean_terminated_length": 689.3056030273438, + "completions/min_length": 125.0, + "completions/min_terminated_length": 125.0, + "epoch": 0.6411197405479219, + "grad_norm": 2.4686570167541504, + "kl": 6.171875, + "learning_rate": 4.0951549574027434e-07, + "loss": 0.3629, + "num_tokens": 1032361662.0, + "reward": 1.869140625, + "reward_std": 0.43076658248901367, + "rewards/accuracy_reward/mean": 0.03125, + "rewards/accuracy_reward/std": 0.17416280508041382, + "rewards/format_reward/mean": 0.888671875, + "rewards/format_reward/std": 0.31484565138816833, + "rewards/tag_count_reward/mean": 0.94921875, + "rewards/tag_count_reward/std": 0.1551761031150818, + "step": 1878 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2033.0, + "completions/mean_length": 772.380859375, + "completions/mean_terminated_length": 720.5263671875, + "completions/min_length": 142.0, + "completions/min_terminated_length": 142.0, + "epoch": 0.6414611248613126, + "grad_norm": 4.238211154937744, + "kl": 9.546875, + "learning_rate": 4.0900628204980924e-07, + "loss": 0.5669, + "num_tokens": 1032825121.0, + "reward": 1.8388671875, + "reward_std": 0.5338704586029053, + "rewards/accuracy_reward/mean": 0.058467742055654526, + "rewards/accuracy_reward/std": 0.23486270010471344, + "rewards/format_reward/mean": 0.859375, + "rewards/format_reward/std": 0.3479743003845215, + "rewards/tag_count_reward/mean": 0.9228515625, + "rewards/tag_count_reward/std": 0.20006181299686432, + "step": 1879 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2018.0, + "completions/mean_length": 771.0703125, + "completions/mean_terminated_length": 737.8035888671875, + "completions/min_length": 117.0, + "completions/min_terminated_length": 117.0, + "epoch": 0.6418025091747034, + "grad_norm": 4.924428939819336, + "kl": 8.28125, + "learning_rate": 4.084972684741386e-07, + "loss": 0.4544, + "num_tokens": 1033305333.0, + "reward": 1.8779296875, + "reward_std": 0.5271910429000854, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.9248046875, + "rewards/tag_count_reward/std": 0.19587218761444092, + "step": 1880 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1922.0, + "completions/mean_length": 780.056640625, + "completions/mean_terminated_length": 749.6260375976562, + "completions/min_length": 183.0, + "completions/min_terminated_length": 183.0, + "epoch": 0.6421438934880942, + "grad_norm": 2.503539800643921, + "kl": 7.171875, + "learning_rate": 4.079884557357142e-07, + "loss": 0.4537, + "num_tokens": 1033781650.0, + "reward": 1.84423828125, + "reward_std": 0.4704209268093109, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.92626953125, + "rewards/tag_count_reward/std": 0.18977881968021393, + "step": 1881 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1980.0, + "completions/mean_length": 775.88671875, + "completions/mean_terminated_length": 742.7454833984375, + "completions/min_length": 190.0, + "completions/min_terminated_length": 190.0, + "epoch": 0.642485277801485, + "grad_norm": 1.1857963800430298, + "kl": 5.984375, + "learning_rate": 4.0747984455670257e-07, + "loss": 0.4052, + "num_tokens": 1034259944.0, + "reward": 1.89892578125, + "reward_std": 0.49622827768325806, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.888671875, + "rewards/format_reward/std": 0.31484565138816833, + "rewards/tag_count_reward/mean": 0.94580078125, + "rewards/tag_count_reward/std": 0.16512742638587952, + "step": 1882 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1895.0, + "completions/mean_length": 749.095703125, + "completions/mean_terminated_length": 723.2211303710938, + "completions/min_length": 13.0, + "completions/min_terminated_length": 13.0, + "epoch": 0.6428266621148758, + "grad_norm": 0.7971708178520203, + "kl": 5.11328125, + "learning_rate": 4.069714356589844e-07, + "loss": 0.3169, + "num_tokens": 1034724873.0, + "reward": 1.91064453125, + "reward_std": 0.49229228496551514, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.890625, + "rewards/format_reward/std": 0.31241437792778015, + "rewards/tag_count_reward/mean": 0.93994140625, + "rewards/tag_count_reward/std": 0.17537043988704681, + "step": 1883 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1744.0, + "completions/mean_length": 760.310546875, + "completions/mean_terminated_length": 724.1104125976562, + "completions/min_length": 62.0, + "completions/min_terminated_length": 62.0, + "epoch": 0.6431680464282666, + "grad_norm": 1.1620593070983887, + "kl": 4.72265625, + "learning_rate": 4.064632297641533e-07, + "loss": 0.3192, + "num_tokens": 1035192024.0, + "reward": 1.939453125, + "reward_std": 0.49195238947868347, + "rewards/accuracy_reward/mean": 0.109375, + "rewards/accuracy_reward/std": 0.31241437792778015, + "rewards/format_reward/mean": 0.888671875, + "rewards/format_reward/std": 0.31484565138816833, + "rewards/tag_count_reward/mean": 0.94140625, + "rewards/tag_count_reward/std": 0.17270830273628235, + "step": 1884 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 796.009765625, + "completions/mean_terminated_length": 763.392822265625, + "completions/min_length": 108.0, + "completions/min_terminated_length": 108.0, + "epoch": 0.6435094307416575, + "grad_norm": 1.8055346012115479, + "kl": 5.65625, + "learning_rate": 4.05955227593514e-07, + "loss": 0.3936, + "num_tokens": 1035681293.0, + "reward": 1.85595703125, + "reward_std": 0.4806082844734192, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.94189453125, + "rewards/tag_count_reward/std": 0.17322689294815063, + "step": 1885 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 755.978515625, + "completions/mean_terminated_length": 711.6060791015625, + "completions/min_length": 158.0, + "completions/min_terminated_length": 158.0, + "epoch": 0.6438508150550483, + "grad_norm": 1.9870210886001587, + "kl": 5.98828125, + "learning_rate": 4.05447429868083e-07, + "loss": 0.431, + "num_tokens": 1036144626.0, + "reward": 1.91650390625, + "reward_std": 0.5447916984558105, + "rewards/accuracy_reward/mean": 0.1171875, + "rewards/accuracy_reward/std": 0.32195815443992615, + "rewards/format_reward/mean": 0.8671875, + "rewards/format_reward/std": 0.33970388770103455, + "rewards/tag_count_reward/mean": 0.93212890625, + "rewards/tag_count_reward/std": 0.18874379992485046, + "step": 1886 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1886.0, + "completions/mean_length": 767.521484375, + "completions/mean_terminated_length": 720.8643798828125, + "completions/min_length": 177.0, + "completions/min_terminated_length": 177.0, + "epoch": 0.644192199368439, + "grad_norm": 1.1415534019470215, + "kl": 5.625, + "learning_rate": 4.049398373085862e-07, + "loss": 0.3773, + "num_tokens": 1036621789.0, + "reward": 1.89111328125, + "reward_std": 0.4812566041946411, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.876953125, + "rewards/format_reward/std": 0.32881227135658264, + "rewards/tag_count_reward/mean": 0.93994140625, + "rewards/tag_count_reward/std": 0.17041848599910736, + "step": 1887 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1968.0, + "completions/mean_length": 773.734375, + "completions/mean_terminated_length": 719.2342529296875, + "completions/min_length": 135.0, + "completions/min_terminated_length": 135.0, + "epoch": 0.6445335836818298, + "grad_norm": 1.1535128355026245, + "kl": 7.625, + "learning_rate": 4.044324506354585e-07, + "loss": 0.5109, + "num_tokens": 1037091765.0, + "reward": 1.796875, + "reward_std": 0.5084168910980225, + "rewards/accuracy_reward/mean": 0.021484375, + "rewards/accuracy_reward/std": 0.14513419568538666, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.923828125, + "rewards/tag_count_reward/std": 0.19859731197357178, + "step": 1888 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2002.0, + "completions/mean_length": 708.26953125, + "completions/mean_terminated_length": 665.0524291992188, + "completions/min_length": 101.0, + "completions/min_terminated_length": 101.0, + "epoch": 0.6448749679952206, + "grad_norm": 1.4276211261749268, + "kl": 7.2265625, + "learning_rate": 4.0392527056884254e-07, + "loss": 0.447, + "num_tokens": 1037532111.0, + "reward": 1.80419921875, + "reward_std": 0.5305805206298828, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.92138671875, + "rewards/tag_count_reward/std": 0.19918283820152283, + "step": 1889 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1990.0, + "completions/mean_length": 757.015625, + "completions/mean_terminated_length": 707.2616577148438, + "completions/min_length": 78.0, + "completions/min_terminated_length": 78.0, + "epoch": 0.6452163523086114, + "grad_norm": 1.2638360261917114, + "kl": 6.40625, + "learning_rate": 4.034182978285877e-07, + "loss": 0.4309, + "num_tokens": 1037990503.0, + "reward": 1.86181640625, + "reward_std": 0.46222448348999023, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.880859375, + "rewards/format_reward/std": 0.32427072525024414, + "rewards/tag_count_reward/mean": 0.93798828125, + "rewards/tag_count_reward/std": 0.17815442383289337, + "step": 1890 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1881.0, + "completions/mean_length": 763.83203125, + "completions/mean_terminated_length": 740.8548583984375, + "completions/min_length": 38.0, + "completions/min_terminated_length": 38.0, + "epoch": 0.6455577366220022, + "grad_norm": 1.6150527000427246, + "kl": 5.5234375, + "learning_rate": 4.0291153313424874e-07, + "loss": 0.3192, + "num_tokens": 1038459409.0, + "reward": 1.8896484375, + "reward_std": 0.4685768783092499, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.880859375, + "rewards/format_reward/std": 0.32427072525024414, + "rewards/tag_count_reward/mean": 0.9404296875, + "rewards/tag_count_reward/std": 0.18000927567481995, + "step": 1891 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1884.0, + "completions/mean_length": 783.49609375, + "completions/mean_terminated_length": 740.0687255859375, + "completions/min_length": 156.0, + "completions/min_terminated_length": 156.0, + "epoch": 0.645899120935393, + "grad_norm": 1.8988356590270996, + "kl": 6.3125, + "learning_rate": 4.024049772050857e-07, + "loss": 0.4145, + "num_tokens": 1038938847.0, + "reward": 1.84765625, + "reward_std": 0.5264095664024353, + "rewards/accuracy_reward/mean": 0.05040322616696358, + "rewards/accuracy_reward/std": 0.21899642050266266, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.9296875, + "rewards/tag_count_reward/std": 0.18752038478851318, + "step": 1892 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.056640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1951.0, + "completions/mean_length": 806.35546875, + "completions/mean_terminated_length": 731.805419921875, + "completions/min_length": 30.0, + "completions/min_terminated_length": 30.0, + "epoch": 0.6462405052487838, + "grad_norm": 2.275977849960327, + "kl": 7.921875, + "learning_rate": 4.01898630760062e-07, + "loss": 0.4934, + "num_tokens": 1039434949.0, + "reward": 1.82470703125, + "reward_std": 0.5624501705169678, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.91064453125, + "rewards/tag_count_reward/std": 0.2154705673456192, + "step": 1893 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1924.0, + "completions/mean_length": 782.603515625, + "completions/mean_terminated_length": 741.7842407226562, + "completions/min_length": 221.0, + "completions/min_terminated_length": 221.0, + "epoch": 0.6465818895621747, + "grad_norm": 1.3620362281799316, + "kl": 6.9453125, + "learning_rate": 4.0139249451784383e-07, + "loss": 0.4666, + "num_tokens": 1039908922.0, + "reward": 1.86181640625, + "reward_std": 0.5229591131210327, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.93212890625, + "rewards/tag_count_reward/std": 0.1854754239320755, + "step": 1894 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1941.0, + "completions/mean_length": 822.70703125, + "completions/mean_terminated_length": 783.1814575195312, + "completions/min_length": 153.0, + "completions/min_terminated_length": 153.0, + "epoch": 0.6469232738755654, + "grad_norm": 2.305196762084961, + "kl": 6.9453125, + "learning_rate": 4.00886569196799e-07, + "loss": 0.4086, + "num_tokens": 1040410996.0, + "reward": 1.86083984375, + "reward_std": 0.5651666522026062, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.91748046875, + "rewards/tag_count_reward/std": 0.2054828256368637, + "step": 1895 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 767.3203125, + "completions/mean_terminated_length": 720.6558837890625, + "completions/min_length": 16.0, + "completions/min_terminated_length": 16.0, + "epoch": 0.6472646581889562, + "grad_norm": 0.988052248954773, + "kl": 5.7578125, + "learning_rate": 4.003808555149961e-07, + "loss": 0.342, + "num_tokens": 1040889832.0, + "reward": 1.8408203125, + "reward_std": 0.5765224695205688, + "rewards/accuracy_reward/mean": 0.099609375, + "rewards/accuracy_reward/std": 0.29977133870124817, + "rewards/format_reward/mean": 0.830078125, + "rewards/format_reward/std": 0.3759314715862274, + "rewards/tag_count_reward/mean": 0.9111328125, + "rewards/tag_count_reward/std": 0.21195456385612488, + "step": 1896 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1802.0, + "completions/mean_length": 736.21875, + "completions/mean_terminated_length": 707.4171752929688, + "completions/min_length": 32.0, + "completions/min_terminated_length": 32.0, + "epoch": 0.647606042502347, + "grad_norm": 2.5826499462127686, + "kl": 5.265625, + "learning_rate": 3.9987535419020303e-07, + "loss": 0.3584, + "num_tokens": 1041344600.0, + "reward": 1.9052734375, + "reward_std": 0.4933793842792511, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.888671875, + "rewards/format_reward/std": 0.31484565138816833, + "rewards/tag_count_reward/mean": 0.9443359375, + "rewards/tag_count_reward/std": 0.1743793785572052, + "step": 1897 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1908.0, + "completions/mean_length": 741.005859375, + "completions/mean_terminated_length": 696.1192016601562, + "completions/min_length": 144.0, + "completions/min_terminated_length": 144.0, + "epoch": 0.6479474268157378, + "grad_norm": 2.448514938354492, + "kl": 6.5078125, + "learning_rate": 3.993700659398863e-07, + "loss": 0.4556, + "num_tokens": 1041799115.0, + "reward": 1.845703125, + "reward_std": 0.4928697347640991, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.875, + "rewards/format_reward/std": 0.3310423493385315, + "rewards/tag_count_reward/mean": 0.927734375, + "rewards/tag_count_reward/std": 0.18937622010707855, + "step": 1898 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2029.0, + "completions/mean_length": 840.2734375, + "completions/mean_terminated_length": 803.8229370117188, + "completions/min_length": 167.0, + "completions/min_terminated_length": 167.0, + "epoch": 0.6482888111291286, + "grad_norm": 1.5282440185546875, + "kl": 7.578125, + "learning_rate": 3.9886499148121055e-07, + "loss": 0.5177, + "num_tokens": 1042306039.0, + "reward": 1.81982421875, + "reward_std": 0.5226191282272339, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.92919921875, + "rewards/tag_count_reward/std": 0.18766237795352936, + "step": 1899 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1991.0, + "completions/mean_length": 786.603515625, + "completions/mean_terminated_length": 745.9132690429688, + "completions/min_length": 178.0, + "completions/min_terminated_length": 178.0, + "epoch": 0.6486301954425194, + "grad_norm": 1.0351145267486572, + "kl": 5.6015625, + "learning_rate": 3.9836013153103643e-07, + "loss": 0.3619, + "num_tokens": 1042789308.0, + "reward": 1.92578125, + "reward_std": 0.5136724710464478, + "rewards/accuracy_reward/mean": 0.109375, + "rewards/accuracy_reward/std": 0.31241437792778015, + "rewards/format_reward/mean": 0.876953125, + "rewards/format_reward/std": 0.32881227135658264, + "rewards/tag_count_reward/mean": 0.939453125, + "rewards/tag_count_reward/std": 0.16988569498062134, + "step": 1900 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2012.0, + "completions/mean_length": 761.123046875, + "completions/mean_terminated_length": 714.2327880859375, + "completions/min_length": 148.0, + "completions/min_terminated_length": 148.0, + "epoch": 0.6489715797559102, + "grad_norm": 1.6239062547683716, + "kl": 7.546875, + "learning_rate": 3.9785548680592027e-07, + "loss": 0.4746, + "num_tokens": 1043254859.0, + "reward": 1.86669921875, + "reward_std": 0.5596586465835571, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.92529296875, + "rewards/tag_count_reward/std": 0.19512130320072174, + "step": 1901 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1979.0, + "completions/mean_length": 842.279296875, + "completions/mean_terminated_length": 780.384033203125, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "epoch": 0.6493129640693011, + "grad_norm": 1.9798287153244019, + "kl": 7.1015625, + "learning_rate": 3.97351058022113e-07, + "loss": 0.4551, + "num_tokens": 1043762282.0, + "reward": 1.82177734375, + "reward_std": 0.5535703897476196, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.8359375, + "rewards/format_reward/std": 0.37069445848464966, + "rewards/tag_count_reward/mean": 0.91552734375, + "rewards/tag_count_reward/std": 0.20167623460292816, + "step": 1902 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 842.6171875, + "completions/mean_terminated_length": 791.0631713867188, + "completions/min_length": 109.0, + "completions/min_terminated_length": 109.0, + "epoch": 0.6496543483826919, + "grad_norm": 1.2850828170776367, + "kl": 6.90625, + "learning_rate": 3.9684684589555894e-07, + "loss": 0.4394, + "num_tokens": 1044273286.0, + "reward": 1.826171875, + "reward_std": 0.5970179438591003, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.82421875, + "rewards/format_reward/std": 0.3810062110424042, + "rewards/tag_count_reward/mean": 0.91015625, + "rewards/tag_count_reward/std": 0.20686452090740204, + "step": 1903 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 793.005859375, + "completions/mean_terminated_length": 757.724853515625, + "completions/min_length": 138.0, + "completions/min_terminated_length": 138.0, + "epoch": 0.6499957326960826, + "grad_norm": 0.9929895401000977, + "kl": 6.6015625, + "learning_rate": 3.9634285114189505e-07, + "loss": 0.4152, + "num_tokens": 1044758633.0, + "reward": 1.828125, + "reward_std": 0.5333633422851562, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.841796875, + "rewards/format_reward/std": 0.36528825759887695, + "rewards/tag_count_reward/mean": 0.9296875, + "rewards/tag_count_reward/std": 0.18155530095100403, + "step": 1904 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1879.0, + "completions/mean_length": 802.458984375, + "completions/mean_terminated_length": 772.5660400390625, + "completions/min_length": 118.0, + "completions/min_terminated_length": 118.0, + "epoch": 0.6503371170094734, + "grad_norm": 1.1284902095794678, + "kl": 4.95703125, + "learning_rate": 3.958390744764497e-07, + "loss": 0.284, + "num_tokens": 1045247860.0, + "reward": 1.83544921875, + "reward_std": 0.4603354036808014, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.93505859375, + "rewards/tag_count_reward/std": 0.1798466295003891, + "step": 1905 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2031.0, + "completions/mean_length": 837.228515625, + "completions/mean_terminated_length": 772.4547119140625, + "completions/min_length": 155.0, + "completions/min_terminated_length": 155.0, + "epoch": 0.6506785013228642, + "grad_norm": 1.227197289466858, + "kl": 8.1171875, + "learning_rate": 3.953355166142417e-07, + "loss": 0.511, + "num_tokens": 1045754345.0, + "reward": 1.7978515625, + "reward_std": 0.567955732345581, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.82421875, + "rewards/format_reward/std": 0.3810062110424042, + "rewards/tag_count_reward/mean": 0.9091796875, + "rewards/tag_count_reward/std": 0.21227891743183136, + "step": 1906 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1933.0, + "completions/mean_length": 765.3984375, + "completions/mean_terminated_length": 726.6881103515625, + "completions/min_length": 156.0, + "completions/min_terminated_length": 156.0, + "epoch": 0.651019885636255, + "grad_norm": 2.1376044750213623, + "kl": 4.8125, + "learning_rate": 3.9483217826997927e-07, + "loss": 0.3393, + "num_tokens": 1046224533.0, + "reward": 1.8876953125, + "reward_std": 0.49774402379989624, + "rewards/accuracy_reward/mean": 0.06666667014360428, + "rewards/accuracy_reward/std": 0.24970406293869019, + "rewards/format_reward/mean": 0.8828125, + "rewards/format_reward/std": 0.32195815443992615, + "rewards/tag_count_reward/mean": 0.9423828125, + "rewards/tag_count_reward/std": 0.17374257743358612, + "step": 1907 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2028.0, + "completions/mean_length": 811.185546875, + "completions/mean_terminated_length": 755.6550903320312, + "completions/min_length": 144.0, + "completions/min_terminated_length": 144.0, + "epoch": 0.6513612699496458, + "grad_norm": 1.065099835395813, + "kl": 7.1796875, + "learning_rate": 3.9432906015805946e-07, + "loss": 0.4794, + "num_tokens": 1046711876.0, + "reward": 1.84423828125, + "reward_std": 0.577957272529602, + "rewards/accuracy_reward/mean": 0.099609375, + "rewards/accuracy_reward/std": 0.29977133870124817, + "rewards/format_reward/mean": 0.83203125, + "rewards/format_reward/std": 0.374204158782959, + "rewards/tag_count_reward/mean": 0.91259765625, + "rewards/tag_count_reward/std": 0.21513772010803223, + "step": 1908 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1959.0, + "completions/mean_length": 822.78515625, + "completions/mean_terminated_length": 788.34130859375, + "completions/min_length": 186.0, + "completions/min_terminated_length": 186.0, + "epoch": 0.6517026542630366, + "grad_norm": 1.9318523406982422, + "kl": 5.5546875, + "learning_rate": 3.93826162992566e-07, + "loss": 0.3595, + "num_tokens": 1047217734.0, + "reward": 1.8408203125, + "reward_std": 0.5040803551673889, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.9287109375, + "rewards/tag_count_reward/std": 0.18845312297344208, + "step": 1909 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1972.0, + "completions/mean_length": 727.7421875, + "completions/mean_terminated_length": 696.0560302734375, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "epoch": 0.6520440385764275, + "grad_norm": 1.0456809997558594, + "kl": 4.80078125, + "learning_rate": 3.933234874872695e-07, + "loss": 0.3013, + "num_tokens": 1047669410.0, + "reward": 1.88232421875, + "reward_std": 0.4403984546661377, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.884765625, + "rewards/format_reward/std": 0.3196168541908264, + "rewards/tag_count_reward/mean": 0.93701171875, + "rewards/tag_count_reward/std": 0.17849735915660858, + "step": 1910 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 795.82421875, + "completions/mean_terminated_length": 760.6224365234375, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "epoch": 0.6523854228898183, + "grad_norm": 1.4011372327804565, + "kl": 6.1640625, + "learning_rate": 3.92821034355626e-07, + "loss": 0.4124, + "num_tokens": 1048152824.0, + "reward": 1.83642578125, + "reward_std": 0.5365915298461914, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.92626953125, + "rewards/tag_count_reward/std": 0.1904222071170807, + "step": 1911 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1974.0, + "completions/mean_length": 747.33984375, + "completions/mean_terminated_length": 710.7750854492188, + "completions/min_length": 142.0, + "completions/min_terminated_length": 142.0, + "epoch": 0.652726807203209, + "grad_norm": 1.0503382682800293, + "kl": 7.578125, + "learning_rate": 3.923188043107758e-07, + "loss": 0.4758, + "num_tokens": 1048608470.0, + "reward": 1.87451171875, + "reward_std": 0.5311700701713562, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.92919921875, + "rewards/tag_count_reward/std": 0.18700948357582092, + "step": 1912 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2006.0, + "completions/mean_length": 820.09765625, + "completions/mean_terminated_length": 770.1829223632812, + "completions/min_length": 131.0, + "completions/min_terminated_length": 131.0, + "epoch": 0.6530681915165998, + "grad_norm": 1.3700637817382812, + "kl": 6.6484375, + "learning_rate": 3.9181679806554267e-07, + "loss": 0.4261, + "num_tokens": 1049103336.0, + "reward": 1.87939453125, + "reward_std": 0.4995768070220947, + "rewards/accuracy_reward/mean": 0.07258064299821854, + "rewards/accuracy_reward/std": 0.25970885157585144, + "rewards/format_reward/mean": 0.880859375, + "rewards/format_reward/std": 0.32427072525024414, + "rewards/tag_count_reward/mean": 0.92822265625, + "rewards/tag_count_reward/std": 0.19684240221977234, + "step": 1913 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1922.0, + "completions/mean_length": 763.25, + "completions/mean_terminated_length": 727.1325073242188, + "completions/min_length": 102.0, + "completions/min_terminated_length": 102.0, + "epoch": 0.6534095758299906, + "grad_norm": 1.0656079053878784, + "kl": 6.4765625, + "learning_rate": 3.913150163324326e-07, + "loss": 0.3954, + "num_tokens": 1049569656.0, + "reward": 1.84521484375, + "reward_std": 0.5555644631385803, + "rewards/accuracy_reward/mean": 0.07459677755832672, + "rewards/accuracy_reward/std": 0.263004869222641, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.92529296875, + "rewards/tag_count_reward/std": 0.18939577043056488, + "step": 1914 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1966.0, + "completions/mean_length": 777.037109375, + "completions/mean_terminated_length": 717.2576293945312, + "completions/min_length": 186.0, + "completions/min_terminated_length": 186.0, + "epoch": 0.6537509601433814, + "grad_norm": 1.1124001741409302, + "kl": 7.5234375, + "learning_rate": 3.90813459823633e-07, + "loss": 0.4902, + "num_tokens": 1050042699.0, + "reward": 1.79638671875, + "reward_std": 0.5529178380966187, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.83203125, + "rewards/format_reward/std": 0.374204158782959, + "rewards/tag_count_reward/mean": 0.91162109375, + "rewards/tag_count_reward/std": 0.21013160049915314, + "step": 1915 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1998.0, + "completions/mean_length": 742.1875, + "completions/mean_terminated_length": 702.776611328125, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "epoch": 0.6540923444567722, + "grad_norm": 1.7929461002349854, + "kl": 5.6171875, + "learning_rate": 3.9031212925101144e-07, + "loss": 0.3663, + "num_tokens": 1050501563.0, + "reward": 1.8544921875, + "reward_std": 0.5551683902740479, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.9228515625, + "rewards/tag_count_reward/std": 0.2006722390651703, + "step": 1916 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 840.861328125, + "completions/mean_terminated_length": 786.6632080078125, + "completions/min_length": 213.0, + "completions/min_terminated_length": 213.0, + "epoch": 0.654433728770163, + "grad_norm": 0.7772568464279175, + "kl": 6.671875, + "learning_rate": 3.898110253261151e-07, + "loss": 0.4309, + "num_tokens": 1051008548.0, + "reward": 1.85986328125, + "reward_std": 0.4899458885192871, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.876953125, + "rewards/format_reward/std": 0.32881227135658264, + "rewards/tag_count_reward/mean": 0.93017578125, + "rewards/tag_count_reward/std": 0.2024376094341278, + "step": 1917 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1724.0, + "completions/mean_length": 816.435546875, + "completions/mean_terminated_length": 758.5091552734375, + "completions/min_length": 140.0, + "completions/min_terminated_length": 140.0, + "epoch": 0.6547751130835539, + "grad_norm": 1.9527724981307983, + "kl": 5.3125, + "learning_rate": 3.8931014876016944e-07, + "loss": 0.3757, + "num_tokens": 1051504435.0, + "reward": 1.87890625, + "reward_std": 0.465145468711853, + "rewards/accuracy_reward/mean": 0.060483869165182114, + "rewards/accuracy_reward/std": 0.2386218160390854, + "rewards/format_reward/mean": 0.87890625, + "rewards/format_reward/std": 0.3265552520751953, + "rewards/tag_count_reward/mean": 0.94140625, + "rewards/tag_count_reward/std": 0.17270830273628235, + "step": 1918 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1868.0, + "completions/mean_length": 788.767578125, + "completions/mean_terminated_length": 748.1471557617188, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "epoch": 0.6551164973969447, + "grad_norm": 1.0534942150115967, + "kl": 4.74609375, + "learning_rate": 3.88809500264077e-07, + "loss": 0.2982, + "num_tokens": 1051987836.0, + "reward": 1.87109375, + "reward_std": 0.5006071329116821, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.935546875, + "rewards/tag_count_reward/std": 0.18036192655563354, + "step": 1919 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 765.619140625, + "completions/mean_terminated_length": 726.9154663085938, + "completions/min_length": 154.0, + "completions/min_terminated_length": 154.0, + "epoch": 0.6554578817103354, + "grad_norm": 1.3416204452514648, + "kl": 5.171875, + "learning_rate": 3.8830908054841673e-07, + "loss": 0.3329, + "num_tokens": 1052455305.0, + "reward": 1.83203125, + "reward_std": 0.5414870977401733, + "rewards/accuracy_reward/mean": 0.058467742055654526, + "rewards/accuracy_reward/std": 0.23486268520355225, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.9296875, + "rewards/tag_count_reward/std": 0.17608344554901123, + "step": 1920 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1873.0, + "completions/mean_length": 784.564453125, + "completions/mean_terminated_length": 746.4325561523438, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.6557992660237262, + "grad_norm": 1.4465922117233276, + "kl": 6.9921875, + "learning_rate": 3.878088903234431e-07, + "loss": 0.415, + "num_tokens": 1052933530.0, + "reward": 1.78955078125, + "reward_std": 0.556403398513794, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.828125, + "rewards/format_reward/std": 0.3776407241821289, + "rewards/tag_count_reward/mean": 0.91650390625, + "rewards/tag_count_reward/std": 0.20086915791034698, + "step": 1921 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2007.0, + "completions/mean_length": 861.740234375, + "completions/mean_terminated_length": 818.5162353515625, + "completions/min_length": 4.0, + "completions/min_terminated_length": 4.0, + "epoch": 0.656140650337117, + "grad_norm": 2.138991594314575, + "kl": 7.296875, + "learning_rate": 3.873089302990844e-07, + "loss": 0.4315, + "num_tokens": 1053455509.0, + "reward": 1.81689453125, + "reward_std": 0.535413384437561, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.841796875, + "rewards/format_reward/std": 0.36528825759887695, + "rewards/tag_count_reward/mean": 0.92236328125, + "rewards/tag_count_reward/std": 0.1964779943227768, + "step": 1922 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2004.0, + "completions/mean_length": 882.025390625, + "completions/mean_terminated_length": 817.115478515625, + "completions/min_length": 123.0, + "completions/min_terminated_length": 123.0, + "epoch": 0.6564820346505078, + "grad_norm": 3.2062578201293945, + "kl": 9.2421875, + "learning_rate": 3.868092011849425e-07, + "loss": 0.549, + "num_tokens": 1053998962.0, + "reward": 1.78662109375, + "reward_std": 0.598289430141449, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.81640625, + "rewards/format_reward/std": 0.3875311613082886, + "rewards/tag_count_reward/mean": 0.90966796875, + "rewards/tag_count_reward/std": 0.21219956874847412, + "step": 1923 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1835.0, + "completions/mean_length": 733.099609375, + "completions/mean_terminated_length": 690.6834716796875, + "completions/min_length": 5.0, + "completions/min_terminated_length": 5.0, + "epoch": 0.6568234189638986, + "grad_norm": 1.269345998764038, + "kl": 6.4453125, + "learning_rate": 3.8630970369029146e-07, + "loss": 0.3937, + "num_tokens": 1054444997.0, + "reward": 1.84033203125, + "reward_std": 0.5486959218978882, + "rewards/accuracy_reward/mean": 0.05645161122083664, + "rewards/accuracy_reward/std": 0.23102475702762604, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.92431640625, + "rewards/tag_count_reward/std": 0.19724006950855255, + "step": 1924 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 815.125, + "completions/mean_terminated_length": 772.7838745117188, + "completions/min_length": 164.0, + "completions/min_terminated_length": 164.0, + "epoch": 0.6571648032772894, + "grad_norm": 1.5820393562316895, + "kl": 7.1328125, + "learning_rate": 3.858104385240768e-07, + "loss": 0.441, + "num_tokens": 1054937733.0, + "reward": 1.83935546875, + "reward_std": 0.5692523717880249, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.91552734375, + "rewards/tag_count_reward/std": 0.20106884837150574, + "step": 1925 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 800.107421875, + "completions/mean_terminated_length": 746.7352905273438, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "epoch": 0.6575061875906802, + "grad_norm": 2.613544464111328, + "kl": 7.4140625, + "learning_rate": 3.85311406394914e-07, + "loss": 0.4794, + "num_tokens": 1055423292.0, + "reward": 1.7880859375, + "reward_std": 0.5436952114105225, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.830078125, + "rewards/format_reward/std": 0.3759314715862274, + "rewards/tag_count_reward/mean": 0.9150390625, + "rewards/tag_count_reward/std": 0.20418420433998108, + "step": 1926 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 793.6015625, + "completions/mean_terminated_length": 758.3373413085938, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "epoch": 0.6578475719040711, + "grad_norm": 1.9729578495025635, + "kl": 7.0078125, + "learning_rate": 3.848126080110878e-07, + "loss": 0.4253, + "num_tokens": 1055903920.0, + "reward": 1.859375, + "reward_std": 0.5435269474983215, + "rewards/accuracy_reward/mean": 0.103515625, + "rewards/accuracy_reward/std": 0.30492907762527466, + "rewards/format_reward/mean": 0.8359375, + "rewards/format_reward/std": 0.37069445848464966, + "rewards/tag_count_reward/mean": 0.919921875, + "rewards/tag_count_reward/std": 0.19074371457099915, + "step": 1927 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1954.0, + "completions/mean_length": 808.990234375, + "completions/mean_terminated_length": 745.3860473632812, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "epoch": 0.6581889562174618, + "grad_norm": 1.095579743385315, + "kl": 8.6328125, + "learning_rate": 3.8431404408055133e-07, + "loss": 0.5712, + "num_tokens": 1056406987.0, + "reward": 1.791015625, + "reward_std": 0.6053373217582703, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.8203125, + "rewards/format_reward/std": 0.38430243730545044, + "rewards/tag_count_reward/mean": 0.904296875, + "rewards/tag_count_reward/std": 0.2203473001718521, + "step": 1928 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1945.0, + "completions/mean_length": 776.359375, + "completions/mean_terminated_length": 702.7933349609375, + "completions/min_length": 128.0, + "completions/min_terminated_length": 128.0, + "epoch": 0.6585303405308526, + "grad_norm": 1.4815394878387451, + "kl": 7.609375, + "learning_rate": 3.8381571531092496e-07, + "loss": 0.4877, + "num_tokens": 1056887987.0, + "reward": 1.78564453125, + "reward_std": 0.5911592245101929, + "rewards/accuracy_reward/mean": 0.08064515888690948, + "rewards/accuracy_reward/std": 0.2725643217563629, + "rewards/format_reward/mean": 0.80859375, + "rewards/format_reward/std": 0.3937928080558777, + "rewards/tag_count_reward/mean": 0.89892578125, + "rewards/tag_count_reward/std": 0.22154901921749115, + "step": 1929 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.060546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1923.0, + "completions/mean_length": 831.947265625, + "completions/mean_terminated_length": 753.5737915039062, + "completions/min_length": 165.0, + "completions/min_terminated_length": 165.0, + "epoch": 0.6588717248442434, + "grad_norm": 2.4619951248168945, + "kl": 7.9765625, + "learning_rate": 3.8331762240949503e-07, + "loss": 0.5882, + "num_tokens": 1057392616.0, + "reward": 1.76904296875, + "reward_std": 0.5942692756652832, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.81640625, + "rewards/format_reward/std": 0.3875311613082886, + "rewards/tag_count_reward/mean": 0.90771484375, + "rewards/tag_count_reward/std": 0.2204204648733139, + "step": 1930 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 857.029296875, + "completions/mean_terminated_length": 803.55712890625, + "completions/min_length": 69.0, + "completions/min_terminated_length": 69.0, + "epoch": 0.6592131091576342, + "grad_norm": 1.31281578540802, + "kl": 6.1484375, + "learning_rate": 3.8281976608321366e-07, + "loss": 0.3631, + "num_tokens": 1057910903.0, + "reward": 1.779296875, + "reward_std": 0.5633202791213989, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.81640625, + "rewards/format_reward/std": 0.3875311613082886, + "rewards/tag_count_reward/mean": 0.916015625, + "rewards/tag_count_reward/std": 0.19666332006454468, + "step": 1931 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1971.0, + "completions/mean_length": 718.65625, + "completions/mean_terminated_length": 684.0240478515625, + "completions/min_length": 145.0, + "completions/min_terminated_length": 145.0, + "epoch": 0.659554493471025, + "grad_norm": 0.8023258447647095, + "kl": 4.671875, + "learning_rate": 3.823221470386965e-07, + "loss": 0.2573, + "num_tokens": 1058351991.0, + "reward": 1.8837890625, + "reward_std": 0.5155116319656372, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.859375, + "rewards/format_reward/std": 0.3479743003845215, + "rewards/tag_count_reward/mean": 0.9267578125, + "rewards/tag_count_reward/std": 0.18835169076919556, + "step": 1932 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1982.0, + "completions/mean_length": 796.8125, + "completions/mean_terminated_length": 732.5831909179688, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "epoch": 0.6598958777844158, + "grad_norm": 1.6733766794204712, + "kl": 6.53515625, + "learning_rate": 3.81824765982223e-07, + "loss": 0.4237, + "num_tokens": 1058835591.0, + "reward": 1.78076171875, + "reward_std": 0.5668002367019653, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.82421875, + "rewards/format_reward/std": 0.3810062110424042, + "rewards/tag_count_reward/mean": 0.91162109375, + "rewards/tag_count_reward/std": 0.2089642435312271, + "step": 1933 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 786.013671875, + "completions/mean_terminated_length": 750.5361328125, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "epoch": 0.6602372620978066, + "grad_norm": 4.714256763458252, + "kl": 5.07421875, + "learning_rate": 3.8132762361973456e-07, + "loss": 0.3631, + "num_tokens": 1059313134.0, + "reward": 1.87939453125, + "reward_std": 0.5234352350234985, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.8671875, + "rewards/format_reward/std": 0.33970388770103455, + "rewards/tag_count_reward/mean": 0.93603515625, + "rewards/tag_count_reward/std": 0.177461177110672, + "step": 1934 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 828.1015625, + "completions/mean_terminated_length": 773.33056640625, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "epoch": 0.6605786464111975, + "grad_norm": 1.5419610738754272, + "kl": 6.0703125, + "learning_rate": 3.8083072065683373e-07, + "loss": 0.3605, + "num_tokens": 1059813762.0, + "reward": 1.76513671875, + "reward_std": 0.558008074760437, + "rewards/accuracy_reward/mean": 0.04032257944345474, + "rewards/accuracy_reward/std": 0.19691328704357147, + "rewards/format_reward/mean": 0.8125, + "rewards/format_reward/std": 0.39069411158561707, + "rewards/tag_count_reward/mean": 0.91357421875, + "rewards/tag_count_reward/std": 0.20506387948989868, + "step": 1935 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 774.724609375, + "completions/mean_terminated_length": 717.55712890625, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "epoch": 0.6609200307245882, + "grad_norm": 1.3766438961029053, + "kl": 5.578125, + "learning_rate": 3.803340577987834e-07, + "loss": 0.3549, + "num_tokens": 1060287365.0, + "reward": 1.87255859375, + "reward_std": 0.5209040641784668, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.92333984375, + "rewards/tag_count_reward/std": 0.1974821388721466, + "step": 1936 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1939.0, + "completions/mean_length": 831.072265625, + "completions/mean_terminated_length": 768.6016845703125, + "completions/min_length": 75.0, + "completions/min_terminated_length": 75.0, + "epoch": 0.661261415037979, + "grad_norm": 1.6967791318893433, + "kl": 6.421875, + "learning_rate": 3.7983763575050575e-07, + "loss": 0.4519, + "num_tokens": 1060789130.0, + "reward": 1.853515625, + "reward_std": 0.5324754118919373, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.857421875, + "rewards/format_reward/std": 0.3499840497970581, + "rewards/tag_count_reward/mean": 0.931640625, + "rewards/tag_count_reward/std": 0.186283141374588, + "step": 1937 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1930.0, + "completions/mean_length": 832.05078125, + "completions/mean_terminated_length": 767.0, + "completions/min_length": 163.0, + "completions/min_terminated_length": 163.0, + "epoch": 0.6616027993513698, + "grad_norm": 1.568750262260437, + "kl": 8.0390625, + "learning_rate": 3.79341455216581e-07, + "loss": 0.5526, + "num_tokens": 1061298772.0, + "reward": 1.83056640625, + "reward_std": 0.579119861125946, + "rewards/accuracy_reward/mean": 0.07258064299821854, + "rewards/accuracy_reward/std": 0.25970885157585144, + "rewards/format_reward/mean": 0.841796875, + "rewards/format_reward/std": 0.36528825759887695, + "rewards/tag_count_reward/mean": 0.91845703125, + "rewards/tag_count_reward/std": 0.20823590457439423, + "step": 1938 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2014.0, + "completions/mean_length": 828.86328125, + "completions/mean_terminated_length": 792.068359375, + "completions/min_length": 157.0, + "completions/min_terminated_length": 157.0, + "epoch": 0.6619441836647606, + "grad_norm": 3.181682825088501, + "kl": 8.484375, + "learning_rate": 3.788455169012469e-07, + "loss": 0.4863, + "num_tokens": 1061797150.0, + "reward": 1.7841796875, + "reward_std": 0.5454611778259277, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.81640625, + "rewards/format_reward/std": 0.3875311613082886, + "rewards/tag_count_reward/mean": 0.9189453125, + "rewards/tag_count_reward/std": 0.19414739310741425, + "step": 1939 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1995.0, + "completions/mean_length": 754.0625, + "completions/mean_terminated_length": 725.6527099609375, + "completions/min_length": 171.0, + "completions/min_terminated_length": 171.0, + "epoch": 0.6622855679781514, + "grad_norm": 0.8705515265464783, + "kl": 6.1875, + "learning_rate": 3.783498215083967e-07, + "loss": 0.3681, + "num_tokens": 1062253006.0, + "reward": 1.875, + "reward_std": 0.4800451993942261, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.8828125, + "rewards/format_reward/std": 0.32195815443992615, + "rewards/tag_count_reward/mean": 0.94140625, + "rewards/tag_count_reward/std": 0.1816815733909607, + "step": 1940 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1983.0, + "completions/mean_length": 795.724609375, + "completions/mean_terminated_length": 768.2295532226562, + "completions/min_length": 183.0, + "completions/min_terminated_length": 183.0, + "epoch": 0.6626269522915422, + "grad_norm": 1.591826319694519, + "kl": 6.5, + "learning_rate": 3.778543697415797e-07, + "loss": 0.3956, + "num_tokens": 1062733809.0, + "reward": 1.82763671875, + "reward_std": 0.5358124375343323, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.857421875, + "rewards/format_reward/std": 0.3499840497970581, + "rewards/tag_count_reward/mean": 0.92333984375, + "rewards/tag_count_reward/std": 0.19686181843280792, + "step": 1941 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 760.849609375, + "completions/mean_terminated_length": 724.6646118164062, + "completions/min_length": 161.0, + "completions/min_terminated_length": 161.0, + "epoch": 0.662968336604933, + "grad_norm": 1.7542084455490112, + "kl": 6.140625, + "learning_rate": 3.77359162303999e-07, + "loss": 0.3728, + "num_tokens": 1063192404.0, + "reward": 1.91455078125, + "reward_std": 0.47773921489715576, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.888671875, + "rewards/format_reward/std": 0.31484565138816833, + "rewards/tag_count_reward/mean": 0.94189453125, + "rewards/tag_count_reward/std": 0.1739315241575241, + "step": 1942 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2001.0, + "completions/mean_length": 811.21484375, + "completions/mean_terminated_length": 778.9940185546875, + "completions/min_length": 112.0, + "completions/min_terminated_length": 112.0, + "epoch": 0.6633097209183239, + "grad_norm": 1.9097706079483032, + "kl": 5.69921875, + "learning_rate": 3.7686419989851104e-07, + "loss": 0.3206, + "num_tokens": 1063690578.0, + "reward": 1.90087890625, + "reward_std": 0.5013086795806885, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.93408203125, + "rewards/tag_count_reward/std": 0.18353331089019775, + "step": 1943 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2019.0, + "completions/mean_length": 792.306640625, + "completions/mean_terminated_length": 751.8003540039062, + "completions/min_length": 101.0, + "completions/min_terminated_length": 101.0, + "epoch": 0.6636511052317146, + "grad_norm": 2.448972463607788, + "kl": 7.4765625, + "learning_rate": 3.763694832276242e-07, + "loss": 0.4391, + "num_tokens": 1064166671.0, + "reward": 1.8828125, + "reward_std": 0.5133916735649109, + "rewards/accuracy_reward/mean": 0.099609375, + "rewards/accuracy_reward/std": 0.29977133870124817, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.927734375, + "rewards/tag_count_reward/std": 0.18677493929862976, + "step": 1944 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2023.0, + "completions/mean_length": 770.896484375, + "completions/mean_terminated_length": 740.2460327148438, + "completions/min_length": 138.0, + "completions/min_terminated_length": 138.0, + "epoch": 0.6639924895451054, + "grad_norm": 1.7370452880859375, + "kl": 7.4453125, + "learning_rate": 3.758750129934988e-07, + "loss": 0.4535, + "num_tokens": 1064637146.0, + "reward": 1.806640625, + "reward_std": 0.5249841213226318, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.927734375, + "rewards/tag_count_reward/std": 0.18677493929862976, + "step": 1945 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1971.0, + "completions/mean_length": 795.15625, + "completions/mean_terminated_length": 754.741943359375, + "completions/min_length": 110.0, + "completions/min_terminated_length": 110.0, + "epoch": 0.6643338738584962, + "grad_norm": 2.202965497970581, + "kl": 7.3125, + "learning_rate": 3.753807898979442e-07, + "loss": 0.437, + "num_tokens": 1065125418.0, + "reward": 1.7822265625, + "reward_std": 0.5233513116836548, + "rewards/accuracy_reward/mean": 0.02734375, + "rewards/accuracy_reward/std": 0.16324250400066376, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.9169921875, + "rewards/tag_count_reward/std": 0.20379072427749634, + "step": 1946 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1990.0, + "completions/mean_length": 794.1875, + "completions/mean_terminated_length": 758.9397583007812, + "completions/min_length": 131.0, + "completions/min_terminated_length": 131.0, + "epoch": 0.664675258171887, + "grad_norm": 1.1559278964996338, + "kl": 6.0390625, + "learning_rate": 3.748868146424201e-07, + "loss": 0.3858, + "num_tokens": 1065612154.0, + "reward": 1.80908203125, + "reward_std": 0.5231696963310242, + "rewards/accuracy_reward/mean": 0.037109375, + "rewards/accuracy_reward/std": 0.18921469151973724, + "rewards/format_reward/mean": 0.841796875, + "rewards/format_reward/std": 0.36528825759887695, + "rewards/tag_count_reward/mean": 0.93017578125, + "rewards/tag_count_reward/std": 0.18341873586177826, + "step": 1947 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2014.0, + "completions/mean_length": 806.630859375, + "completions/mean_terminated_length": 763.9979858398438, + "completions/min_length": 218.0, + "completions/min_terminated_length": 218.0, + "epoch": 0.6650166424852778, + "grad_norm": 1.417567491531372, + "kl": 5.375, + "learning_rate": 3.7439308792803405e-07, + "loss": 0.3436, + "num_tokens": 1066102669.0, + "reward": 1.86669921875, + "reward_std": 0.56736159324646, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.849609375, + "rewards/format_reward/std": 0.35780346393585205, + "rewards/tag_count_reward/mean": 0.92724609375, + "rewards/tag_count_reward/std": 0.19079817831516266, + "step": 1948 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1984.0, + "completions/mean_length": 790.591796875, + "completions/mean_terminated_length": 747.4081420898438, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "epoch": 0.6653580267986686, + "grad_norm": 1.6329373121261597, + "kl": 6.6953125, + "learning_rate": 3.738996104555406e-07, + "loss": 0.4305, + "num_tokens": 1066585612.0, + "reward": 1.861328125, + "reward_std": 0.591179370880127, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.919921875, + "rewards/tag_count_reward/std": 0.20013105869293213, + "step": 1949 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2025.0, + "completions/mean_length": 863.728515625, + "completions/mean_terminated_length": 813.0774536132812, + "completions/min_length": 129.0, + "completions/min_terminated_length": 129.0, + "epoch": 0.6656994111120594, + "grad_norm": 2.3752756118774414, + "kl": 6.1328125, + "learning_rate": 3.734063829253411e-07, + "loss": 0.4053, + "num_tokens": 1067110257.0, + "reward": 1.8310546875, + "reward_std": 0.5816721320152283, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.9169921875, + "rewards/tag_count_reward/std": 0.20617742836475372, + "step": 1950 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2002.0, + "completions/mean_length": 811.49609375, + "completions/mean_terminated_length": 755.9795532226562, + "completions/min_length": 31.0, + "completions/min_terminated_length": 31.0, + "epoch": 0.6660407954254502, + "grad_norm": 0.8571250438690186, + "kl": 7.1328125, + "learning_rate": 3.7291340603748146e-07, + "loss": 0.452, + "num_tokens": 1067602031.0, + "reward": 1.80322265625, + "reward_std": 0.5911970138549805, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.810546875, + "rewards/format_reward/std": 0.3922513723373413, + "rewards/tag_count_reward/mean": 0.91259765625, + "rewards/tag_count_reward/std": 0.19981031119823456, + "step": 1951 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1949.0, + "completions/mean_length": 732.78515625, + "completions/mean_terminated_length": 690.3588256835938, + "completions/min_length": 89.0, + "completions/min_terminated_length": 89.0, + "epoch": 0.666382179738841, + "grad_norm": 1.4395986795425415, + "kl": 5.8984375, + "learning_rate": 3.724206804916526e-07, + "loss": 0.3438, + "num_tokens": 1068055121.0, + "reward": 1.83740234375, + "reward_std": 0.5787259340286255, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.82421875, + "rewards/format_reward/std": 0.3810062110424042, + "rewards/tag_count_reward/mean": 0.91748046875, + "rewards/tag_count_reward/std": 0.19697342813014984, + "step": 1952 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1946.0, + "completions/mean_length": 736.90234375, + "completions/mean_terminated_length": 708.1157836914062, + "completions/min_length": 131.0, + "completions/min_terminated_length": 131.0, + "epoch": 0.6667235640522318, + "grad_norm": 1.1494282484054565, + "kl": 6.9609375, + "learning_rate": 3.7192820698718797e-07, + "loss": 0.4307, + "num_tokens": 1068510511.0, + "reward": 1.810546875, + "reward_std": 0.5527421236038208, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.82421875, + "rewards/format_reward/std": 0.3810062110424042, + "rewards/tag_count_reward/mean": 0.91796875, + "rewards/tag_count_reward/std": 0.196245014667511, + "step": 1953 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1989.0, + "completions/mean_length": 801.71484375, + "completions/mean_terminated_length": 743.0960693359375, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "epoch": 0.6670649483656226, + "grad_norm": 1.0540169477462769, + "kl": 6.69140625, + "learning_rate": 3.7143598622306374e-07, + "loss": 0.4275, + "num_tokens": 1069007277.0, + "reward": 1.79931640625, + "reward_std": 0.5230346918106079, + "rewards/accuracy_reward/mean": 0.029296875, + "rewards/accuracy_reward/std": 0.16880230605602264, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.91845703125, + "rewards/tag_count_reward/std": 0.2022770643234253, + "step": 1954 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1999.0, + "completions/mean_length": 863.84765625, + "completions/mean_terminated_length": 800.4979248046875, + "completions/min_length": 8.0, + "completions/min_terminated_length": 8.0, + "epoch": 0.6674063326790134, + "grad_norm": 2.0837478637695312, + "kl": 8.3515625, + "learning_rate": 3.7094401889789715e-07, + "loss": 0.4981, + "num_tokens": 1069527295.0, + "reward": 1.7197265625, + "reward_std": 0.63157057762146, + "rewards/accuracy_reward/mean": 0.037109375, + "rewards/accuracy_reward/std": 0.18921469151973724, + "rewards/format_reward/mean": 0.794921875, + "rewards/format_reward/std": 0.4041535556316376, + "rewards/tag_count_reward/mean": 0.8876953125, + "rewards/tag_count_reward/std": 0.23425979912281036, + "step": 1955 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1978.0, + "completions/mean_length": 807.505859375, + "completions/mean_terminated_length": 777.7340087890625, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.6677477169924042, + "grad_norm": 1.1107139587402344, + "kl": 6.5859375, + "learning_rate": 3.70452305709946e-07, + "loss": 0.3762, + "num_tokens": 1070018066.0, + "reward": 1.7998046875, + "reward_std": 0.5736995935440063, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.822265625, + "rewards/format_reward/std": 0.3826628625392914, + "rewards/tag_count_reward/mean": 0.9130859375, + "rewards/tag_count_reward/std": 0.2075263112783432, + "step": 1956 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 812.60546875, + "completions/mean_terminated_length": 767.5911254882812, + "completions/min_length": 154.0, + "completions/min_terminated_length": 154.0, + "epoch": 0.668089101305795, + "grad_norm": 1.0080444812774658, + "kl": 7.5703125, + "learning_rate": 3.699608473571072e-07, + "loss": 0.5046, + "num_tokens": 1070511496.0, + "reward": 1.7890625, + "reward_std": 0.541830837726593, + "rewards/accuracy_reward/mean": 0.033203125, + "rewards/accuracy_reward/std": 0.17934183776378632, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.916015625, + "rewards/tag_count_reward/std": 0.2087314873933792, + "step": 1957 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1912.0, + "completions/mean_length": 826.48828125, + "completions/mean_terminated_length": 779.4117431640625, + "completions/min_length": 117.0, + "completions/min_terminated_length": 117.0, + "epoch": 0.6684304856191858, + "grad_norm": 0.8483101725578308, + "kl": 6.21875, + "learning_rate": 3.694696445369161e-07, + "loss": 0.3647, + "num_tokens": 1071007442.0, + "reward": 1.8505859375, + "reward_std": 0.5570029020309448, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.9248046875, + "rewards/tag_count_reward/std": 0.19773660600185394, + "step": 1958 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1886.0, + "completions/mean_length": 801.98828125, + "completions/mean_terminated_length": 753.967529296875, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "epoch": 0.6687718699325766, + "grad_norm": 2.572129249572754, + "kl": 5.078125, + "learning_rate": 3.68978697946545e-07, + "loss": 0.3551, + "num_tokens": 1071490860.0, + "reward": 1.8896484375, + "reward_std": 0.4518565237522125, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.89453125, + "rewards/format_reward/std": 0.3074568510055542, + "rewards/tag_count_reward/mean": 0.9404296875, + "rewards/tag_count_reward/std": 0.18270690739154816, + "step": 1959 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2021.0, + "completions/mean_length": 810.318359375, + "completions/mean_terminated_length": 783.1437377929688, + "completions/min_length": 70.0, + "completions/min_terminated_length": 70.0, + "epoch": 0.6691132542459673, + "grad_norm": 1.489697813987732, + "kl": 5.30859375, + "learning_rate": 3.6848800828280303e-07, + "loss": 0.3124, + "num_tokens": 1071993823.0, + "reward": 1.83935546875, + "reward_std": 0.5098040699958801, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.92529296875, + "rewards/tag_count_reward/std": 0.19512130320072174, + "step": 1960 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1879.0, + "completions/mean_length": 810.70703125, + "completions/mean_terminated_length": 763.0222778320312, + "completions/min_length": 149.0, + "completions/min_terminated_length": 149.0, + "epoch": 0.6694546385593582, + "grad_norm": 1.9213225841522217, + "kl": 6.4765625, + "learning_rate": 3.67997576242134e-07, + "loss": 0.4127, + "num_tokens": 1072488105.0, + "reward": 1.84521484375, + "reward_std": 0.5162262320518494, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.92138671875, + "rewards/tag_count_reward/std": 0.20101657509803772, + "step": 1961 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1895.0, + "completions/mean_length": 756.341796875, + "completions/mean_terminated_length": 722.69140625, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "epoch": 0.669796022872749, + "grad_norm": 0.9990555047988892, + "kl": 5.2890625, + "learning_rate": 3.67507402520617e-07, + "loss": 0.3254, + "num_tokens": 1072949272.0, + "reward": 1.962890625, + "reward_std": 0.476193904876709, + "rewards/accuracy_reward/mean": 0.1411290317773819, + "rewards/accuracy_reward/std": 0.3485061228275299, + "rewards/format_reward/mean": 0.884765625, + "rewards/format_reward/std": 0.3196168541908264, + "rewards/tag_count_reward/mean": 0.94140625, + "rewards/tag_count_reward/std": 0.17759640514850616, + "step": 1962 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 758.56640625, + "completions/mean_terminated_length": 732.8804931640625, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "epoch": 0.6701374071861398, + "grad_norm": 0.6265716552734375, + "kl": 4.88671875, + "learning_rate": 3.6701748781396367e-07, + "loss": 0.3077, + "num_tokens": 1073412394.0, + "reward": 1.927734375, + "reward_std": 0.4050692915916443, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.8984375, + "rewards/format_reward/std": 0.30236753821372986, + "rewards/tag_count_reward/mean": 0.94921875, + "rewards/tag_count_reward/std": 0.1628674566745758, + "step": 1963 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1857.0, + "completions/mean_length": 819.044921875, + "completions/mean_terminated_length": 753.29833984375, + "completions/min_length": 153.0, + "completions/min_terminated_length": 153.0, + "epoch": 0.6704787914995306, + "grad_norm": 4.422807216644287, + "kl": 9.9765625, + "learning_rate": 3.6652783281751873e-07, + "loss": 0.576, + "num_tokens": 1073909841.0, + "reward": 1.76220703125, + "reward_std": 0.6102453470230103, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.81640625, + "rewards/format_reward/std": 0.3875311613082886, + "rewards/tag_count_reward/mean": 0.90283203125, + "rewards/tag_count_reward/std": 0.2210913747549057, + "step": 1964 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1990.0, + "completions/mean_length": 798.322265625, + "completions/mean_terminated_length": 750.1602172851562, + "completions/min_length": 117.0, + "completions/min_terminated_length": 117.0, + "epoch": 0.6708201758129214, + "grad_norm": 3.0986688137054443, + "kl": 8.03125, + "learning_rate": 3.6603843822625734e-07, + "loss": 0.4542, + "num_tokens": 1074403590.0, + "reward": 1.85205078125, + "reward_std": 0.6030385494232178, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.83203125, + "rewards/format_reward/std": 0.374204158782959, + "rewards/tag_count_reward/mean": 0.91455078125, + "rewards/tag_count_reward/std": 0.20666059851646423, + "step": 1965 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1956.0, + "completions/mean_length": 854.54296875, + "completions/mean_terminated_length": 811.0567016601562, + "completions/min_length": 126.0, + "completions/min_terminated_length": 126.0, + "epoch": 0.6711615601263122, + "grad_norm": 2.9957141876220703, + "kl": 7.7109375, + "learning_rate": 3.6554930473478595e-07, + "loss": 0.4545, + "num_tokens": 1074912140.0, + "reward": 1.8427734375, + "reward_std": 0.530123233795166, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.9248046875, + "rewards/tag_count_reward/std": 0.19335831701755524, + "step": 1966 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 828.880859375, + "completions/mean_terminated_length": 802.11376953125, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "epoch": 0.671502944439703, + "grad_norm": 0.9075517058372498, + "kl": 5.890625, + "learning_rate": 3.650604330373398e-07, + "loss": 0.3298, + "num_tokens": 1075415743.0, + "reward": 1.88134765625, + "reward_std": 0.515263557434082, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.93408203125, + "rewards/tag_count_reward/std": 0.17812223732471466, + "step": 1967 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2005.0, + "completions/mean_length": 820.759765625, + "completions/mean_terminated_length": 781.1713256835938, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "epoch": 0.6718443287530937, + "grad_norm": 0.6880972981452942, + "kl": 5.3203125, + "learning_rate": 3.6457182382778315e-07, + "loss": 0.3174, + "num_tokens": 1075910580.0, + "reward": 1.8984375, + "reward_std": 0.49258172512054443, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.876953125, + "rewards/format_reward/std": 0.32881227135658264, + "rewards/tag_count_reward/mean": 0.9375, + "rewards/tag_count_reward/std": 0.18104934692382812, + "step": 1968 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 845.310546875, + "completions/mean_terminated_length": 811.5, + "completions/min_length": 168.0, + "completions/min_terminated_length": 168.0, + "epoch": 0.6721857130664846, + "grad_norm": 2.171827793121338, + "kl": 5.7890625, + "learning_rate": 3.6408347779960734e-07, + "loss": 0.3805, + "num_tokens": 1076421923.0, + "reward": 1.8515625, + "reward_std": 0.5119212865829468, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.875, + "rewards/format_reward/std": 0.3310423493385315, + "rewards/tag_count_reward/mean": 0.93359375, + "rewards/tag_count_reward/std": 0.18894177675247192, + "step": 1969 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1944.0, + "completions/mean_length": 853.38671875, + "completions/mean_terminated_length": 802.2933349609375, + "completions/min_length": 173.0, + "completions/min_terminated_length": 173.0, + "epoch": 0.6725270973798754, + "grad_norm": 0.7126238346099854, + "kl": 6.28125, + "learning_rate": 3.6359539564593036e-07, + "loss": 0.3868, + "num_tokens": 1076944089.0, + "reward": 1.8876953125, + "reward_std": 0.4900527000427246, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.87890625, + "rewards/format_reward/std": 0.3265552520751953, + "rewards/tag_count_reward/mean": 0.9326171875, + "rewards/tag_count_reward/std": 0.19244687259197235, + "step": 1970 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1983.0, + "completions/mean_length": 827.697265625, + "completions/mean_terminated_length": 790.8671875, + "completions/min_length": 164.0, + "completions/min_terminated_length": 164.0, + "epoch": 0.6728684816932662, + "grad_norm": 1.5118255615234375, + "kl": 4.75, + "learning_rate": 3.631075780594952e-07, + "loss": 0.3059, + "num_tokens": 1077437662.0, + "reward": 1.88525390625, + "reward_std": 0.4044254422187805, + "rewards/accuracy_reward/mean": 0.037109375, + "rewards/accuracy_reward/std": 0.18921469151973724, + "rewards/format_reward/mean": 0.900390625, + "rewards/format_reward/std": 0.29977133870124817, + "rewards/tag_count_reward/mean": 0.94775390625, + "rewards/tag_count_reward/std": 0.16795603930950165, + "step": 1971 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2031.0, + "completions/mean_length": 866.716796875, + "completions/mean_terminated_length": 800.9546508789062, + "completions/min_length": 191.0, + "completions/min_terminated_length": 191.0, + "epoch": 0.673209866006657, + "grad_norm": 0.9211564660072327, + "kl": 6.71875, + "learning_rate": 3.626200257326697e-07, + "loss": 0.4365, + "num_tokens": 1077963629.0, + "reward": 1.8642578125, + "reward_std": 0.5401686429977417, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.86328125, + "rewards/format_reward/std": 0.3438861668109894, + "rewards/tag_count_reward/mean": 0.9228515625, + "rewards/tag_count_reward/std": 0.20429649949073792, + "step": 1972 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1961.0, + "completions/mean_length": 827.431640625, + "completions/mean_terminated_length": 788.0584716796875, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "epoch": 0.6735512503200478, + "grad_norm": 0.977889895439148, + "kl": 6.328125, + "learning_rate": 3.621327393574458e-07, + "loss": 0.3933, + "num_tokens": 1078466554.0, + "reward": 1.9169921875, + "reward_std": 0.5646554231643677, + "rewards/accuracy_reward/mean": 0.1171875, + "rewards/accuracy_reward/std": 0.32195815443992615, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.9345703125, + "rewards/tag_count_reward/std": 0.18000927567481995, + "step": 1973 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1996.0, + "completions/mean_length": 856.5234375, + "completions/mean_terminated_length": 797.9262084960938, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "epoch": 0.6738926346334386, + "grad_norm": 1.3242242336273193, + "kl": 6.3203125, + "learning_rate": 3.616457196254367e-07, + "loss": 0.4215, + "num_tokens": 1078991414.0, + "reward": 1.82958984375, + "reward_std": 0.485893577337265, + "rewards/accuracy_reward/mean": 0.03515625, + "rewards/accuracy_reward/std": 0.1843547374010086, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.92919921875, + "rewards/tag_count_reward/std": 0.19532686471939087, + "step": 1974 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1934.0, + "completions/mean_length": 838.375, + "completions/mean_terminated_length": 801.8671875, + "completions/min_length": 140.0, + "completions/min_terminated_length": 140.0, + "epoch": 0.6742340189468294, + "grad_norm": 1.0018812417984009, + "kl": 5.53515625, + "learning_rate": 3.6115896722787833e-07, + "loss": 0.3668, + "num_tokens": 1079503526.0, + "reward": 1.89794921875, + "reward_std": 0.4896755814552307, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.884765625, + "rewards/format_reward/std": 0.3196168541908264, + "rewards/tag_count_reward/mean": 0.94091796875, + "rewards/tag_count_reward/std": 0.1721867471933365, + "step": 1975 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1988.0, + "completions/mean_length": 793.333984375, + "completions/mean_terminated_length": 752.86083984375, + "completions/min_length": 115.0, + "completions/min_terminated_length": 115.0, + "epoch": 0.6745754032602201, + "grad_norm": 1.165183424949646, + "kl": 6.2578125, + "learning_rate": 3.606724828556265e-07, + "loss": 0.3995, + "num_tokens": 1079994033.0, + "reward": 1.82861328125, + "reward_std": 0.4689714014530182, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.859375, + "rewards/format_reward/std": 0.3479743003845215, + "rewards/tag_count_reward/mean": 0.93017578125, + "rewards/tag_count_reward/std": 0.18737702071666718, + "step": 1976 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 782.591796875, + "completions/mean_terminated_length": 747.01806640625, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "epoch": 0.674916787573611, + "grad_norm": 1.5107274055480957, + "kl": 6.8046875, + "learning_rate": 3.6018626719915646e-07, + "loss": 0.431, + "num_tokens": 1080471184.0, + "reward": 1.88037109375, + "reward_std": 0.5095371007919312, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.87890625, + "rewards/format_reward/std": 0.3265552520751953, + "rewards/tag_count_reward/mean": 0.93896484375, + "rewards/tag_count_reward/std": 0.17917592823505402, + "step": 1977 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 766.013671875, + "completions/mean_terminated_length": 729.973876953125, + "completions/min_length": 153.0, + "completions/min_terminated_length": 153.0, + "epoch": 0.6752581718870018, + "grad_norm": 2.550865650177002, + "kl": 8.0859375, + "learning_rate": 3.5970032094856305e-07, + "loss": 0.4875, + "num_tokens": 1080937911.0, + "reward": 1.89404296875, + "reward_std": 0.5230495929718018, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.92919921875, + "rewards/tag_count_reward/std": 0.19280590116977692, + "step": 1978 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1997.0, + "completions/mean_length": 820.462890625, + "completions/mean_terminated_length": 783.4144897460938, + "completions/min_length": 131.0, + "completions/min_terminated_length": 131.0, + "epoch": 0.6755995562003926, + "grad_norm": 1.9703261852264404, + "kl": 7.1484375, + "learning_rate": 3.5921464479355744e-07, + "loss": 0.4242, + "num_tokens": 1081427444.0, + "reward": 1.8330078125, + "reward_std": 0.5129296779632568, + "rewards/accuracy_reward/mean": 0.033203125, + "rewards/accuracy_reward/std": 0.17934183776378632, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.9306640625, + "rewards/tag_count_reward/std": 0.19175048172473907, + "step": 1979 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1816.0, + "completions/mean_length": 772.873046875, + "completions/mean_terminated_length": 721.03857421875, + "completions/min_length": 163.0, + "completions/min_terminated_length": 163.0, + "epoch": 0.6759409405137834, + "grad_norm": 1.5295605659484863, + "kl": 7.34375, + "learning_rate": 3.5872923942346875e-07, + "loss": 0.4682, + "num_tokens": 1081897843.0, + "reward": 1.880859375, + "reward_std": 0.4452829360961914, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.890625, + "rewards/format_reward/std": 0.31241437792778015, + "rewards/tag_count_reward/mean": 0.9453125, + "rewards/tag_count_reward/std": 0.17328274250030518, + "step": 1980 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1985.0, + "completions/mean_length": 805.861328125, + "completions/mean_terminated_length": 770.9417114257812, + "completions/min_length": 54.0, + "completions/min_terminated_length": 54.0, + "epoch": 0.6762823248271742, + "grad_norm": 1.6627914905548096, + "kl": 4.974609375, + "learning_rate": 3.582441055272406e-07, + "loss": 0.3214, + "num_tokens": 1082400764.0, + "reward": 1.93310546875, + "reward_std": 0.5067430734634399, + "rewards/accuracy_reward/mean": 0.111328125, + "rewards/accuracy_reward/std": 0.31484565138816833, + "rewards/format_reward/mean": 0.8828125, + "rewards/format_reward/std": 0.32195815443992615, + "rewards/tag_count_reward/mean": 0.93896484375, + "rewards/tag_count_reward/std": 0.17917592823505402, + "step": 1981 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2014.0, + "completions/mean_length": 818.62109375, + "completions/mean_terminated_length": 778.9636840820312, + "completions/min_length": 177.0, + "completions/min_terminated_length": 177.0, + "epoch": 0.676623709140565, + "grad_norm": 1.4015086889266968, + "kl": 5.55078125, + "learning_rate": 3.577592437934321e-07, + "loss": 0.3658, + "num_tokens": 1082892522.0, + "reward": 1.99462890625, + "reward_std": 0.4820824861526489, + "rewards/accuracy_reward/mean": 0.134765625, + "rewards/accuracy_reward/std": 0.3418070077896118, + "rewards/format_reward/mean": 0.908203125, + "rewards/format_reward/std": 0.289021372795105, + "rewards/tag_count_reward/mean": 0.95166015625, + "rewards/tag_count_reward/std": 0.16694028675556183, + "step": 1982 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1840.0, + "completions/mean_length": 803.07421875, + "completions/mean_terminated_length": 744.5194091796875, + "completions/min_length": 158.0, + "completions/min_terminated_length": 158.0, + "epoch": 0.6769650934539558, + "grad_norm": 2.1695973873138428, + "kl": 6.640625, + "learning_rate": 3.572746549102159e-07, + "loss": 0.4691, + "num_tokens": 1083388384.0, + "reward": 1.86376953125, + "reward_std": 0.4834197163581848, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.884765625, + "rewards/format_reward/std": 0.3196168541908264, + "rewards/tag_count_reward/mean": 0.93603515625, + "rewards/tag_count_reward/std": 0.18620555102825165, + "step": 1983 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1989.0, + "completions/mean_length": 731.607421875, + "completions/mean_terminated_length": 721.2421264648438, + "completions/min_length": 173.0, + "completions/min_terminated_length": 173.0, + "epoch": 0.6773064777673465, + "grad_norm": 1.1995561122894287, + "kl": 2.927734375, + "learning_rate": 3.567903395653767e-07, + "loss": 0.1891, + "num_tokens": 1083837687.0, + "reward": 1.982421875, + "reward_std": 0.43703126907348633, + "rewards/accuracy_reward/mean": 0.11088709533214569, + "rewards/accuracy_reward/std": 0.3143092691898346, + "rewards/format_reward/mean": 0.91796875, + "rewards/format_reward/std": 0.2746807038784027, + "rewards/tag_count_reward/mean": 0.95703125, + "rewards/tag_count_reward/std": 0.1462491750717163, + "step": 1984 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2007.0, + "completions/mean_length": 787.6796875, + "completions/mean_terminated_length": 772.7352294921875, + "completions/min_length": 211.0, + "completions/min_terminated_length": 211.0, + "epoch": 0.6776478620807374, + "grad_norm": 1.6124293804168701, + "kl": 3.671875, + "learning_rate": 3.563062984463123e-07, + "loss": 0.2372, + "num_tokens": 1084316835.0, + "reward": 1.9638671875, + "reward_std": 0.4849792420864105, + "rewards/accuracy_reward/mean": 0.11328125, + "rewards/accuracy_reward/std": 0.3172462284564972, + "rewards/format_reward/mean": 0.900390625, + "rewards/format_reward/std": 0.29977133870124817, + "rewards/tag_count_reward/mean": 0.9501953125, + "rewards/tag_count_reward/std": 0.1593773365020752, + "step": 1985 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1969.0, + "completions/mean_length": 768.302734375, + "completions/mean_terminated_length": 729.6800537109375, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "epoch": 0.6779892463941282, + "grad_norm": 1.0881859064102173, + "kl": 4.890625, + "learning_rate": 3.5582253224003e-07, + "loss": 0.3377, + "num_tokens": 1084791342.0, + "reward": 1.8779296875, + "reward_std": 0.4652717411518097, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.8828125, + "rewards/format_reward/std": 0.32195815443992615, + "rewards/tag_count_reward/mean": 0.9365234375, + "rewards/tag_count_reward/std": 0.1779806911945343, + "step": 1986 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 799.8203125, + "completions/mean_terminated_length": 754.340087890625, + "completions/min_length": 56.0, + "completions/min_terminated_length": 56.0, + "epoch": 0.678330630707519, + "grad_norm": 1.7563881874084473, + "kl": 6.609375, + "learning_rate": 3.553390416331478e-07, + "loss": 0.4096, + "num_tokens": 1085282130.0, + "reward": 1.8115234375, + "reward_std": 0.5413820147514343, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.9248046875, + "rewards/tag_count_reward/std": 0.19145125150680542, + "step": 1987 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2010.0, + "completions/mean_length": 832.158203125, + "completions/mean_terminated_length": 769.7433471679688, + "completions/min_length": 138.0, + "completions/min_terminated_length": 138.0, + "epoch": 0.6786720150209098, + "grad_norm": 2.784532070159912, + "kl": 8.6328125, + "learning_rate": 3.5485582731189176e-07, + "loss": 0.4979, + "num_tokens": 1085784931.0, + "reward": 1.7998046875, + "reward_std": 0.5651649236679077, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.828125, + "rewards/format_reward/std": 0.3776407241821289, + "rewards/tag_count_reward/mean": 0.9072265625, + "rewards/tag_count_reward/std": 0.216009259223938, + "step": 1988 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2033.0, + "completions/mean_length": 808.9921875, + "completions/mean_terminated_length": 774.1605834960938, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "epoch": 0.6790133993343006, + "grad_norm": 1.4216352701187134, + "kl": 6.53125, + "learning_rate": 3.5437288996209704e-07, + "loss": 0.3949, + "num_tokens": 1086277039.0, + "reward": 1.86474609375, + "reward_std": 0.5679696202278137, + "rewards/accuracy_reward/mean": 0.087890625, + "rewards/accuracy_reward/std": 0.2834126651287079, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.92529296875, + "rewards/tag_count_reward/std": 0.19068296253681183, + "step": 1989 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2006.0, + "completions/mean_length": 789.94921875, + "completions/mean_terminated_length": 738.8088989257812, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "epoch": 0.6793547836476914, + "grad_norm": 3.0501508712768555, + "kl": 8.9375, + "learning_rate": 3.5389023026920384e-07, + "loss": 0.5632, + "num_tokens": 1086764821.0, + "reward": 1.80712890625, + "reward_std": 0.5999290943145752, + "rewards/accuracy_reward/mean": 0.087890625, + "rewards/accuracy_reward/std": 0.2834126651287079, + "rewards/format_reward/mean": 0.81640625, + "rewards/format_reward/std": 0.3875311613082886, + "rewards/tag_count_reward/mean": 0.90283203125, + "rewards/tag_count_reward/std": 0.22274482250213623, + "step": 1990 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 793.982421875, + "completions/mean_terminated_length": 743.0060424804688, + "completions/min_length": 32.0, + "completions/min_terminated_length": 32.0, + "epoch": 0.6796961679610822, + "grad_norm": 3.246222972869873, + "kl": 8.07421875, + "learning_rate": 3.534078489182598e-07, + "loss": 0.4707, + "num_tokens": 1087252092.0, + "reward": 1.74365234375, + "reward_std": 0.5731247663497925, + "rewards/accuracy_reward/mean": 0.033203125, + "rewards/accuracy_reward/std": 0.17934183776378632, + "rewards/format_reward/mean": 0.806640625, + "rewards/format_reward/std": 0.39531853795051575, + "rewards/tag_count_reward/mean": 0.90380859375, + "rewards/tag_count_reward/std": 0.21364909410476685, + "step": 1991 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1972.0, + "completions/mean_length": 746.060546875, + "completions/mean_terminated_length": 695.8843383789062, + "completions/min_length": 78.0, + "completions/min_terminated_length": 78.0, + "epoch": 0.6800375522744729, + "grad_norm": 2.1648812294006348, + "kl": 7.1484375, + "learning_rate": 3.5292574659391716e-07, + "loss": 0.4566, + "num_tokens": 1087704699.0, + "reward": 1.8671875, + "reward_std": 0.5629680752754211, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.849609375, + "rewards/format_reward/std": 0.35780346393585205, + "rewards/tag_count_reward/mean": 0.92578125, + "rewards/tag_count_reward/std": 0.1937359869480133, + "step": 1992 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.072265625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1998.0, + "completions/mean_length": 822.86328125, + "completions/mean_terminated_length": 727.4315185546875, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "epoch": 0.6803789365878637, + "grad_norm": 3.0087924003601074, + "kl": 9.5234375, + "learning_rate": 3.524439239804318e-07, + "loss": 0.6276, + "num_tokens": 1088205829.0, + "reward": 1.78857421875, + "reward_std": 0.6144096851348877, + "rewards/accuracy_reward/mean": 0.1015625, + "rewards/accuracy_reward/std": 0.30236753821372986, + "rewards/format_reward/mean": 0.80078125, + "rewards/format_reward/std": 0.39980348944664, + "rewards/tag_count_reward/mean": 0.88623046875, + "rewards/tag_count_reward/std": 0.2400074452161789, + "step": 1993 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1979.0, + "completions/mean_length": 776.033203125, + "completions/mean_terminated_length": 718.9244384765625, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "epoch": 0.6807203209012546, + "grad_norm": 1.7864247560501099, + "kl": 6.5, + "learning_rate": 3.519623817616629e-07, + "loss": 0.4486, + "num_tokens": 1088685478.0, + "reward": 1.83984375, + "reward_std": 0.5918079614639282, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.841796875, + "rewards/format_reward/std": 0.36528825759887695, + "rewards/tag_count_reward/mean": 0.91796875, + "rewards/tag_count_reward/std": 0.2059757262468338, + "step": 1994 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1996.0, + "completions/mean_length": 761.509765625, + "completions/mean_terminated_length": 717.3273315429688, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "epoch": 0.6810617052146454, + "grad_norm": 1.4981460571289062, + "kl": 5.1953125, + "learning_rate": 3.514811206210716e-07, + "loss": 0.3131, + "num_tokens": 1089149531.0, + "reward": 1.828125, + "reward_std": 0.4942927360534668, + "rewards/accuracy_reward/mean": 0.037109375, + "rewards/accuracy_reward/std": 0.18921469151973724, + "rewards/format_reward/mean": 0.86328125, + "rewards/format_reward/std": 0.3438861668109894, + "rewards/tag_count_reward/mean": 0.927734375, + "rewards/tag_count_reward/std": 0.1919422447681427, + "step": 1995 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 801.904296875, + "completions/mean_terminated_length": 771.998046875, + "completions/min_length": 156.0, + "completions/min_terminated_length": 156.0, + "epoch": 0.6814030895280362, + "grad_norm": 2.0844836235046387, + "kl": 5.75, + "learning_rate": 3.5100014124171995e-07, + "loss": 0.3838, + "num_tokens": 1089641050.0, + "reward": 1.833984375, + "reward_std": 0.5666632652282715, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.923828125, + "rewards/tag_count_reward/std": 0.19297493994235992, + "step": 1996 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2004.0, + "completions/mean_length": 791.677734375, + "completions/mean_terminated_length": 727.184814453125, + "completions/min_length": 167.0, + "completions/min_terminated_length": 167.0, + "epoch": 0.681744473841427, + "grad_norm": 2.7286179065704346, + "kl": 6.75, + "learning_rate": 3.505194443062701e-07, + "loss": 0.4735, + "num_tokens": 1090122053.0, + "reward": 1.830078125, + "reward_std": 0.5621203780174255, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.833984375, + "rewards/format_reward/std": 0.3724585771560669, + "rewards/tag_count_reward/mean": 0.90625, + "rewards/tag_count_reward/std": 0.22229015827178955, + "step": 1997 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 729.859375, + "completions/mean_terminated_length": 695.51904296875, + "completions/min_length": 84.0, + "completions/min_terminated_length": 84.0, + "epoch": 0.6820858581548178, + "grad_norm": 3.886467695236206, + "kl": 4.31640625, + "learning_rate": 3.5003903049698356e-07, + "loss": 0.3389, + "num_tokens": 1090564541.0, + "reward": 1.96240234375, + "reward_std": 0.465962290763855, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.904296875, + "rewards/format_reward/std": 0.2944713830947876, + "rewards/tag_count_reward/mean": 0.95263671875, + "rewards/tag_count_reward/std": 0.15188921988010406, + "step": 1998 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2015.0, + "completions/mean_length": 778.80078125, + "completions/mean_terminated_length": 713.6468505859375, + "completions/min_length": 141.0, + "completions/min_terminated_length": 141.0, + "epoch": 0.6824272424682086, + "grad_norm": 1.008063554763794, + "kl": 5.9375, + "learning_rate": 3.4955890049572e-07, + "loss": 0.3869, + "num_tokens": 1091043527.0, + "reward": 1.89892578125, + "reward_std": 0.5463054776191711, + "rewards/accuracy_reward/mean": 0.115234375, + "rewards/accuracy_reward/std": 0.3196168541908264, + "rewards/format_reward/mean": 0.859375, + "rewards/format_reward/std": 0.3479743003845215, + "rewards/tag_count_reward/mean": 0.92431640625, + "rewards/tag_count_reward/std": 0.1966189742088318, + "step": 1999 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1924.0, + "completions/mean_length": 779.732421875, + "completions/mean_terminated_length": 749.2940063476562, + "completions/min_length": 3.0, + "completions/min_terminated_length": 3.0, + "epoch": 0.6827686267815993, + "grad_norm": 1.1527817249298096, + "kl": 6.109375, + "learning_rate": 3.490790549839359e-07, + "loss": 0.388, + "num_tokens": 1091526270.0, + "reward": 1.80908203125, + "reward_std": 0.5509384274482727, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.8359375, + "rewards/format_reward/std": 0.37069445848464966, + "rewards/tag_count_reward/mean": 0.90673828125, + "rewards/tag_count_reward/std": 0.2177731692790985, + "step": 2000 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1989.0, + "completions/mean_length": 773.5, + "completions/mean_terminated_length": 748.111572265625, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "epoch": 0.6831100110949901, + "grad_norm": 1.2709134817123413, + "kl": 5.1953125, + "learning_rate": 3.4859949464268456e-07, + "loss": 0.3416, + "num_tokens": 1092006478.0, + "reward": 1.85986328125, + "reward_std": 0.5381577610969543, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.93017578125, + "rewards/tag_count_reward/std": 0.18997004628181458, + "step": 2001 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1870.0, + "completions/mean_length": 719.9140625, + "completions/mean_terminated_length": 690.7545166015625, + "completions/min_length": 164.0, + "completions/min_terminated_length": 164.0, + "epoch": 0.683451395408381, + "grad_norm": 1.2730306386947632, + "kl": 5.625, + "learning_rate": 3.481202201526136e-07, + "loss": 0.3509, + "num_tokens": 1092455394.0, + "reward": 1.849609375, + "reward_std": 0.5185894966125488, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.857421875, + "rewards/format_reward/std": 0.3499840497970581, + "rewards/tag_count_reward/mean": 0.92578125, + "rewards/tag_count_reward/std": 0.18600594997406006, + "step": 2002 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1986.0, + "completions/mean_length": 779.9140625, + "completions/mean_terminated_length": 728.3658447265625, + "completions/min_length": 128.0, + "completions/min_terminated_length": 128.0, + "epoch": 0.6837927797217718, + "grad_norm": 1.204888939857483, + "kl": 7.76171875, + "learning_rate": 3.4764123219396613e-07, + "loss": 0.5224, + "num_tokens": 1092934246.0, + "reward": 1.86328125, + "reward_std": 0.552196741104126, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.9296875, + "rewards/tag_count_reward/std": 0.1835651993751526, + "step": 2003 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1994.0, + "completions/mean_length": 769.638671875, + "completions/mean_terminated_length": 736.3346557617188, + "completions/min_length": 21.0, + "completions/min_terminated_length": 21.0, + "epoch": 0.6841341640351626, + "grad_norm": 1.5956668853759766, + "kl": 7.1171875, + "learning_rate": 3.471625314465773e-07, + "loss": 0.454, + "num_tokens": 1093408269.0, + "reward": 1.8251953125, + "reward_std": 0.5145806074142456, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.9248046875, + "rewards/tag_count_reward/std": 0.18887855112552643, + "step": 2004 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 769.146484375, + "completions/mean_terminated_length": 719.8600463867188, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "epoch": 0.6844755483485534, + "grad_norm": 3.291630744934082, + "kl": 7.890625, + "learning_rate": 3.466841185898756e-07, + "loss": 0.4675, + "num_tokens": 1093893368.0, + "reward": 1.7783203125, + "reward_std": 0.5961418747901917, + "rewards/accuracy_reward/mean": 0.06854838877916336, + "rewards/accuracy_reward/std": 0.25293973088264465, + "rewards/format_reward/mean": 0.80859375, + "rewards/format_reward/std": 0.3937928080558777, + "rewards/tag_count_reward/mean": 0.9033203125, + "rewards/tag_count_reward/std": 0.21485605835914612, + "step": 2005 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2030.0, + "completions/mean_length": 739.6171875, + "completions/mean_terminated_length": 694.682861328125, + "completions/min_length": 78.0, + "completions/min_terminated_length": 78.0, + "epoch": 0.6848169326619442, + "grad_norm": 2.7448348999023438, + "kl": 7.609375, + "learning_rate": 3.4620599430288077e-07, + "loss": 0.507, + "num_tokens": 1094349124.0, + "reward": 1.830078125, + "reward_std": 0.5540704727172852, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.9140625, + "rewards/tag_count_reward/std": 0.2108539640903473, + "step": 2006 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1865.0, + "completions/mean_length": 739.24609375, + "completions/mean_terminated_length": 688.8073120117188, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "epoch": 0.685158316975335, + "grad_norm": 3.8149802684783936, + "kl": 7.921875, + "learning_rate": 3.457281592642026e-07, + "loss": 0.4571, + "num_tokens": 1094811010.0, + "reward": 1.81787109375, + "reward_std": 0.542671799659729, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.849609375, + "rewards/format_reward/std": 0.35780346393585205, + "rewards/tag_count_reward/mean": 0.91943359375, + "rewards/tag_count_reward/std": 0.20506852865219116, + "step": 2007 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1850.0, + "completions/mean_length": 721.427734375, + "completions/mean_terminated_length": 667.5020141601562, + "completions/min_length": 110.0, + "completions/min_terminated_length": 110.0, + "epoch": 0.6854997012887258, + "grad_norm": 1.4170217514038086, + "kl": 6.5859375, + "learning_rate": 3.452506141520406e-07, + "loss": 0.4268, + "num_tokens": 1095256397.0, + "reward": 1.90576171875, + "reward_std": 0.5055143237113953, + "rewards/accuracy_reward/mean": 0.107421875, + "rewards/accuracy_reward/std": 0.30995169281959534, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.92919921875, + "rewards/tag_count_reward/std": 0.19025151431560516, + "step": 2008 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1981.0, + "completions/mean_length": 699.96484375, + "completions/mean_terminated_length": 673.111572265625, + "completions/min_length": 102.0, + "completions/min_terminated_length": 102.0, + "epoch": 0.6858410856021165, + "grad_norm": 1.7245264053344727, + "kl": 5.9765625, + "learning_rate": 3.4477335964418237e-07, + "loss": 0.3983, + "num_tokens": 1095699547.0, + "reward": 1.82568359375, + "reward_std": 0.5196582078933716, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.92529296875, + "rewards/tag_count_reward/std": 0.1900404542684555, + "step": 2009 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2033.0, + "completions/mean_length": 776.666015625, + "completions/mean_terminated_length": 727.6693725585938, + "completions/min_length": 160.0, + "completions/min_terminated_length": 160.0, + "epoch": 0.6861824699155074, + "grad_norm": 1.3633527755737305, + "kl": 6.90625, + "learning_rate": 3.442963964180039e-07, + "loss": 0.4468, + "num_tokens": 1096177952.0, + "reward": 1.81787109375, + "reward_std": 0.5621967315673828, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.91357421875, + "rewards/tag_count_reward/std": 0.21267634630203247, + "step": 2010 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2001.0, + "completions/mean_length": 771.876953125, + "completions/mean_terminated_length": 733.3621215820312, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "epoch": 0.6865238542288982, + "grad_norm": 2.924910545349121, + "kl": 5.1171875, + "learning_rate": 3.4381972515046675e-07, + "loss": 0.3663, + "num_tokens": 1096653425.0, + "reward": 1.86083984375, + "reward_std": 0.5203917026519775, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.93310546875, + "rewards/tag_count_reward/std": 0.18779471516609192, + "step": 2011 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1959.0, + "completions/mean_length": 775.73828125, + "completions/mean_terminated_length": 732.0444946289062, + "completions/min_length": 135.0, + "completions/min_terminated_length": 135.0, + "epoch": 0.686865238542289, + "grad_norm": 3.1730332374572754, + "kl": 5.5703125, + "learning_rate": 3.4334334651811895e-07, + "loss": 0.4103, + "num_tokens": 1097123883.0, + "reward": 1.8857421875, + "reward_std": 0.4755280613899231, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.892578125, + "rewards/format_reward/std": 0.30995169281959534, + "rewards/tag_count_reward/mean": 0.9404296875, + "rewards/tag_count_reward/std": 0.17727059125900269, + "step": 2012 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1900.0, + "completions/mean_length": 767.333984375, + "completions/mean_terminated_length": 717.9776611328125, + "completions/min_length": 178.0, + "completions/min_terminated_length": 178.0, + "epoch": 0.6872066228556798, + "grad_norm": 1.8164947032928467, + "kl": 7.328125, + "learning_rate": 3.428672611970931e-07, + "loss": 0.4735, + "num_tokens": 1097602390.0, + "reward": 1.853515625, + "reward_std": 0.5261906981468201, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.86328125, + "rewards/format_reward/std": 0.3438861668109894, + "rewards/tag_count_reward/mean": 0.92578125, + "rewards/tag_count_reward/std": 0.19872237741947174, + "step": 2013 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1833.0, + "completions/mean_length": 779.359375, + "completions/mean_terminated_length": 722.3999633789062, + "completions/min_length": 161.0, + "completions/min_terminated_length": 161.0, + "epoch": 0.6875480071690706, + "grad_norm": 1.9302737712860107, + "kl": 6.7109375, + "learning_rate": 3.423914698631044e-07, + "loss": 0.4349, + "num_tokens": 1098079854.0, + "reward": 1.85888671875, + "reward_std": 0.5373066663742065, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.92333984375, + "rewards/tag_count_reward/std": 0.20597051084041595, + "step": 2014 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 826.439453125, + "completions/mean_terminated_length": 784.4868774414062, + "completions/min_length": 134.0, + "completions/min_terminated_length": 134.0, + "epoch": 0.6878893914824614, + "grad_norm": 1.0338008403778076, + "kl": 7.3125, + "learning_rate": 3.4191597319145246e-07, + "loss": 0.47, + "num_tokens": 1098572783.0, + "reward": 1.82373046875, + "reward_std": 0.5456644892692566, + "rewards/accuracy_reward/mean": 0.052419353276491165, + "rewards/accuracy_reward/std": 0.22309619188308716, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.92138671875, + "rewards/tag_count_reward/std": 0.20463472604751587, + "step": 2015 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1988.0, + "completions/mean_length": 721.32421875, + "completions/mean_terminated_length": 670.1947021484375, + "completions/min_length": 135.0, + "completions/min_terminated_length": 135.0, + "epoch": 0.6882307757958522, + "grad_norm": 1.4855358600616455, + "kl": 6.5078125, + "learning_rate": 3.414407718570172e-07, + "loss": 0.431, + "num_tokens": 1099023829.0, + "reward": 1.89501953125, + "reward_std": 0.45963796973228455, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.888671875, + "rewards/format_reward/std": 0.31484565138816833, + "rewards/tag_count_reward/mean": 0.94189453125, + "rewards/tag_count_reward/std": 0.17533229291439056, + "step": 2016 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1985.0, + "completions/mean_length": 802.76171875, + "completions/mean_terminated_length": 741.5204467773438, + "completions/min_length": 115.0, + "completions/min_terminated_length": 115.0, + "epoch": 0.6885721601092429, + "grad_norm": 2.0285561084747314, + "kl": 8.4453125, + "learning_rate": 3.4096586653426053e-07, + "loss": 0.5113, + "num_tokens": 1099513595.0, + "reward": 1.84619140625, + "reward_std": 0.5822349786758423, + "rewards/accuracy_reward/mean": 0.103515625, + "rewards/accuracy_reward/std": 0.30492907762527466, + "rewards/format_reward/mean": 0.83203125, + "rewards/format_reward/std": 0.374204158782959, + "rewards/tag_count_reward/mean": 0.91064453125, + "rewards/tag_count_reward/std": 0.21884989738464355, + "step": 2017 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 764.267578125, + "completions/mean_terminated_length": 722.8568115234375, + "completions/min_length": 107.0, + "completions/min_terminated_length": 107.0, + "epoch": 0.6889135444226338, + "grad_norm": 1.34463369846344, + "kl": 7.734375, + "learning_rate": 3.404912578972232e-07, + "loss": 0.4943, + "num_tokens": 1099979332.0, + "reward": 1.83740234375, + "reward_std": 0.5318292379379272, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.92529296875, + "rewards/tag_count_reward/std": 0.19512130320072174, + "step": 2018 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2028.0, + "completions/mean_length": 798.251953125, + "completions/mean_terminated_length": 757.9375, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "epoch": 0.6892549287360246, + "grad_norm": 1.9976649284362793, + "kl": 6.140625, + "learning_rate": 3.400169466195256e-07, + "loss": 0.3996, + "num_tokens": 1100474757.0, + "reward": 1.87255859375, + "reward_std": 0.5219699144363403, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.92919921875, + "rewards/tag_count_reward/std": 0.19719645380973816, + "step": 2019 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2023.0, + "completions/mean_length": 777.501953125, + "completions/mean_terminated_length": 720.4591674804688, + "completions/min_length": 143.0, + "completions/min_terminated_length": 143.0, + "epoch": 0.6895963130494154, + "grad_norm": 1.4548605680465698, + "kl": 7.53125, + "learning_rate": 3.395429333743663e-07, + "loss": 0.4906, + "num_tokens": 1100941478.0, + "reward": 1.81103515625, + "reward_std": 0.5591021776199341, + "rewards/accuracy_reward/mean": 0.05645161122083664, + "rewards/accuracy_reward/std": 0.23102475702762604, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.91259765625, + "rewards/tag_count_reward/std": 0.21399766206741333, + "step": 2020 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1922.0, + "completions/mean_length": 772.6328125, + "completions/mean_terminated_length": 734.1408081054688, + "completions/min_length": 151.0, + "completions/min_terminated_length": 151.0, + "epoch": 0.6899376973628062, + "grad_norm": 1.5992845296859741, + "kl": 5.578125, + "learning_rate": 3.3906921883451957e-07, + "loss": 0.3679, + "num_tokens": 1101418618.0, + "reward": 1.93408203125, + "reward_std": 0.5032426118850708, + "rewards/accuracy_reward/mean": 0.103515625, + "rewards/accuracy_reward/std": 0.30492907762527466, + "rewards/format_reward/mean": 0.890625, + "rewards/format_reward/std": 0.31241437792778015, + "rewards/tag_count_reward/mean": 0.93994140625, + "rewards/tag_count_reward/std": 0.18288137018680573, + "step": 2021 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1930.0, + "completions/mean_length": 774.623046875, + "completions/mean_terminated_length": 736.1911010742188, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "epoch": 0.690279081676197, + "grad_norm": 1.1282154321670532, + "kl": 5.109375, + "learning_rate": 3.3859580367233695e-07, + "loss": 0.3283, + "num_tokens": 1101886553.0, + "reward": 1.87548828125, + "reward_std": 0.41805973649024963, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.88671875, + "rewards/format_reward/std": 0.3172462284564972, + "rewards/tag_count_reward/mean": 0.94189453125, + "rewards/tag_count_reward/std": 0.17322689294815063, + "step": 2022 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1984.0, + "completions/mean_length": 780.873046875, + "completions/mean_terminated_length": 739.9979858398438, + "completions/min_length": 126.0, + "completions/min_terminated_length": 126.0, + "epoch": 0.6906204659895878, + "grad_norm": 1.2273058891296387, + "kl": 5.2109375, + "learning_rate": 3.3812268855974475e-07, + "loss": 0.383, + "num_tokens": 1102353176.0, + "reward": 1.93115234375, + "reward_std": 0.3770022392272949, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.9140625, + "rewards/format_reward/std": 0.28054583072662354, + "rewards/tag_count_reward/mean": 0.95458984375, + "rewards/tag_count_reward/std": 0.1603061705827713, + "step": 2023 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1969.0, + "completions/mean_length": 712.54296875, + "completions/mean_terminated_length": 683.2215576171875, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "epoch": 0.6909618503029786, + "grad_norm": 1.0310503244400024, + "kl": 4.8515625, + "learning_rate": 3.376498741682433e-07, + "loss": 0.3448, + "num_tokens": 1102796814.0, + "reward": 1.96044921875, + "reward_std": 0.45979851484298706, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.91015625, + "rewards/format_reward/std": 0.2862374484539032, + "rewards/tag_count_reward/mean": 0.94482421875, + "rewards/tag_count_reward/std": 0.18106979131698608, + "step": 2024 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1843.0, + "completions/mean_length": 710.173828125, + "completions/mean_terminated_length": 694.310302734375, + "completions/min_length": 165.0, + "completions/min_terminated_length": 165.0, + "epoch": 0.6913032346163693, + "grad_norm": 1.1273776292800903, + "kl": 3.265625, + "learning_rate": 3.3717736116890585e-07, + "loss": 0.2073, + "num_tokens": 1103243111.0, + "reward": 1.93359375, + "reward_std": 0.33209705352783203, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.9296875, + "rewards/format_reward/std": 0.25592297315597534, + "rewards/tag_count_reward/mean": 0.96484375, + "rewards/tag_count_reward/std": 0.14070820808410645, + "step": 2025 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.009765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2007.0, + "completions/mean_length": 705.255859375, + "completions/mean_terminated_length": 692.0137939453125, + "completions/min_length": 152.0, + "completions/min_terminated_length": 152.0, + "epoch": 0.6916446189297601, + "grad_norm": 1.2359204292297363, + "kl": 4.28125, + "learning_rate": 3.3670515023237866e-07, + "loss": 0.2912, + "num_tokens": 1103679482.0, + "reward": 1.98388671875, + "reward_std": 0.459837943315506, + "rewards/accuracy_reward/mean": 0.126953125, + "rewards/accuracy_reward/std": 0.33324605226516724, + "rewards/format_reward/mean": 0.904296875, + "rewards/format_reward/std": 0.2944713830947876, + "rewards/tag_count_reward/mean": 0.95263671875, + "rewards/tag_count_reward/std": 0.15897136926651, + "step": 2026 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1940.0, + "completions/mean_length": 768.39453125, + "completions/mean_terminated_length": 729.7745971679688, + "completions/min_length": 176.0, + "completions/min_terminated_length": 176.0, + "epoch": 0.691986003243151, + "grad_norm": 1.2850275039672852, + "kl": 4.8046875, + "learning_rate": 3.362332420288786e-07, + "loss": 0.3307, + "num_tokens": 1104144388.0, + "reward": 1.9501953125, + "reward_std": 0.4600157141685486, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.912109375, + "rewards/format_reward/std": 0.2834126651287079, + "rewards/tag_count_reward/mean": 0.9541015625, + "rewards/tag_count_reward/std": 0.1551237553358078, + "step": 2027 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1953.0, + "completions/mean_length": 750.505859375, + "completions/mean_terminated_length": 724.6593627929688, + "completions/min_length": 178.0, + "completions/min_terminated_length": 178.0, + "epoch": 0.6923273875565418, + "grad_norm": 1.6819928884506226, + "kl": 4.51171875, + "learning_rate": 3.3576163722819273e-07, + "loss": 0.3185, + "num_tokens": 1104615527.0, + "reward": 1.90869140625, + "reward_std": 0.3826946020126343, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.91796875, + "rewards/format_reward/std": 0.2746807038784027, + "rewards/tag_count_reward/mean": 0.95166015625, + "rewards/tag_count_reward/std": 0.15944552421569824, + "step": 2028 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1996.0, + "completions/mean_length": 725.001953125, + "completions/mean_terminated_length": 706.6633911132812, + "completions/min_length": 156.0, + "completions/min_terminated_length": 156.0, + "epoch": 0.6926687718699326, + "grad_norm": 1.4650315046310425, + "kl": 4.1171875, + "learning_rate": 3.3529033649967843e-07, + "loss": 0.2806, + "num_tokens": 1105076824.0, + "reward": 1.94873046875, + "reward_std": 0.3908160626888275, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.916015625, + "rewards/format_reward/std": 0.2776356339454651, + "rewards/tag_count_reward/mean": 0.95263671875, + "rewards/tag_count_reward/std": 0.15742509067058563, + "step": 2029 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 714.0, + "completions/mean_terminated_length": 698.1818237304688, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "epoch": 0.6930101561833234, + "grad_norm": 1.4474140405654907, + "kl": 4.62109375, + "learning_rate": 3.3481934051226024e-07, + "loss": 0.2668, + "num_tokens": 1105517496.0, + "reward": 1.9580078125, + "reward_std": 0.4199273884296417, + "rewards/accuracy_reward/mean": 0.087890625, + "rewards/accuracy_reward/std": 0.2834126651287079, + "rewards/format_reward/mean": 0.912109375, + "rewards/format_reward/std": 0.2834126651287079, + "rewards/tag_count_reward/mean": 0.9580078125, + "rewards/tag_count_reward/std": 0.14736543595790863, + "step": 2030 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1916.0, + "completions/mean_length": 749.974609375, + "completions/mean_terminated_length": 729.37109375, + "completions/min_length": 4.0, + "completions/min_terminated_length": 4.0, + "epoch": 0.6933515404967142, + "grad_norm": 2.888395309448242, + "kl": 5.88671875, + "learning_rate": 3.3434864993443123e-07, + "loss": 0.3294, + "num_tokens": 1105984731.0, + "reward": 1.89697265625, + "reward_std": 0.4438630938529968, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.900390625, + "rewards/format_reward/std": 0.29977133870124817, + "rewards/tag_count_reward/mean": 0.94775390625, + "rewards/tag_count_reward/std": 0.16795603930950165, + "step": 2031 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1986.0, + "completions/mean_length": 708.150390625, + "completions/mean_terminated_length": 684.1768798828125, + "completions/min_length": 96.0, + "completions/min_terminated_length": 96.0, + "epoch": 0.693692924810105, + "grad_norm": 3.6145339012145996, + "kl": 6.3203125, + "learning_rate": 3.338782654342506e-07, + "loss": 0.3938, + "num_tokens": 1106412920.0, + "reward": 1.9541015625, + "reward_std": 0.42945945262908936, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.91796875, + "rewards/format_reward/std": 0.2746807038784027, + "rewards/tag_count_reward/mean": 0.9521484375, + "rewards/tag_count_reward/std": 0.15997575223445892, + "step": 2032 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2012.0, + "completions/mean_length": 780.103515625, + "completions/mean_terminated_length": 744.4598388671875, + "completions/min_length": 156.0, + "completions/min_terminated_length": 156.0, + "epoch": 0.6940343091234957, + "grad_norm": 8.065526962280273, + "kl": 7.828125, + "learning_rate": 3.334081876793427e-07, + "loss": 0.473, + "num_tokens": 1106899821.0, + "reward": 1.8876953125, + "reward_std": 0.48659005761146545, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.87890625, + "rewards/format_reward/std": 0.3265552520751953, + "rewards/tag_count_reward/mean": 0.9345703125, + "rewards/tag_count_reward/std": 0.18863557279109955, + "step": 2033 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1980.0, + "completions/mean_length": 732.80859375, + "completions/mean_terminated_length": 701.2440185546875, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.6943756934368865, + "grad_norm": 4.281768798828125, + "kl": 8.1953125, + "learning_rate": 3.3293841733689745e-07, + "loss": 0.4801, + "num_tokens": 1107347147.0, + "reward": 1.935546875, + "reward_std": 0.44750791788101196, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.89453125, + "rewards/format_reward/std": 0.3074568510055542, + "rewards/tag_count_reward/mean": 0.9453125, + "rewards/tag_count_reward/std": 0.1660744547843933, + "step": 2034 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1995.0, + "completions/mean_length": 717.529296875, + "completions/mean_terminated_length": 691.0259399414062, + "completions/min_length": 14.0, + "completions/min_terminated_length": 14.0, + "epoch": 0.6947170777502774, + "grad_norm": 5.1386308670043945, + "kl": 8.859375, + "learning_rate": 3.324689550736674e-07, + "loss": 0.5111, + "num_tokens": 1107793066.0, + "reward": 1.9365234375, + "reward_std": 0.5220915079116821, + "rewards/accuracy_reward/mean": 0.130859375, + "rewards/accuracy_reward/std": 0.33757632970809937, + "rewards/format_reward/mean": 0.873046875, + "rewards/format_reward/std": 0.33324605226516724, + "rewards/tag_count_reward/mean": 0.9326171875, + "rewards/tag_count_reward/std": 0.1806451380252838, + "step": 2035 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2033.0, + "completions/mean_length": 790.515625, + "completions/mean_terminated_length": 755.1646118164062, + "completions/min_length": 209.0, + "completions/min_terminated_length": 209.0, + "epoch": 0.6950584620636682, + "grad_norm": 6.330515384674072, + "kl": 9.7578125, + "learning_rate": 3.3199980155596895e-07, + "loss": 0.5453, + "num_tokens": 1108272530.0, + "reward": 1.76953125, + "reward_std": 0.6054547429084778, + "rewards/accuracy_reward/mean": 0.03515625, + "rewards/accuracy_reward/std": 0.1843547374010086, + "rewards/format_reward/mean": 0.828125, + "rewards/format_reward/std": 0.3776407241821289, + "rewards/tag_count_reward/mean": 0.90625, + "rewards/tag_count_reward/std": 0.22338789701461792, + "step": 2036 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1954.0, + "completions/mean_length": 789.607421875, + "completions/mean_terminated_length": 743.7550659179688, + "completions/min_length": 180.0, + "completions/min_terminated_length": 180.0, + "epoch": 0.695399846377059, + "grad_norm": 4.596363067626953, + "kl": 8.5546875, + "learning_rate": 3.315309574496792e-07, + "loss": 0.4735, + "num_tokens": 1108760777.0, + "reward": 1.80078125, + "reward_std": 0.5478556752204895, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.83203125, + "rewards/format_reward/std": 0.374204158782959, + "rewards/tag_count_reward/mean": 0.91796875, + "rewards/tag_count_reward/std": 0.2041865587234497, + "step": 2037 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1995.0, + "completions/mean_length": 757.49609375, + "completions/mean_terminated_length": 726.5240478515625, + "completions/min_length": 127.0, + "completions/min_terminated_length": 127.0, + "epoch": 0.6957412306904498, + "grad_norm": 3.411527633666992, + "kl": 7.25, + "learning_rate": 3.310624234202369e-07, + "loss": 0.408, + "num_tokens": 1109224919.0, + "reward": 1.82080078125, + "reward_std": 0.5130316019058228, + "rewards/accuracy_reward/mean": 0.0463709682226181, + "rewards/accuracy_reward/std": 0.21049949526786804, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.92822265625, + "rewards/tag_count_reward/std": 0.18794220685958862, + "step": 2038 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1985.0, + "completions/mean_length": 754.6484375, + "completions/mean_terminated_length": 723.6080322265625, + "completions/min_length": 123.0, + "completions/min_terminated_length": 123.0, + "epoch": 0.6960826150038406, + "grad_norm": 3.268136739730835, + "kl": 8.4296875, + "learning_rate": 3.305942001326404e-07, + "loss": 0.4984, + "num_tokens": 1109692083.0, + "reward": 1.8564453125, + "reward_std": 0.6011396646499634, + "rewards/accuracy_reward/mean": 0.10282257944345474, + "rewards/accuracy_reward/std": 0.30403366684913635, + "rewards/format_reward/mean": 0.8359375, + "rewards/format_reward/std": 0.37069445848464966, + "rewards/tag_count_reward/mean": 0.9208984375, + "rewards/tag_count_reward/std": 0.19744645059108734, + "step": 2039 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1963.0, + "completions/mean_length": 771.66796875, + "completions/mean_terminated_length": 741.0360107421875, + "completions/min_length": 152.0, + "completions/min_terminated_length": 152.0, + "epoch": 0.6964239993172314, + "grad_norm": 1.0883808135986328, + "kl": 6.703125, + "learning_rate": 3.3012628825144685e-07, + "loss": 0.4257, + "num_tokens": 1110164665.0, + "reward": 1.8818359375, + "reward_std": 0.5368679165840149, + "rewards/accuracy_reward/mean": 0.087890625, + "rewards/accuracy_reward/std": 0.2834126651287079, + "rewards/format_reward/mean": 0.86328125, + "rewards/format_reward/std": 0.3438861668109894, + "rewards/tag_count_reward/mean": 0.9306640625, + "rewards/tag_count_reward/std": 0.19238728284835815, + "step": 2040 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 771.451171875, + "completions/mean_terminated_length": 740.8140258789062, + "completions/min_length": 123.0, + "completions/min_terminated_length": 123.0, + "epoch": 0.6967653836306221, + "grad_norm": 1.7109276056289673, + "kl": 5.6015625, + "learning_rate": 3.296586884407717e-07, + "loss": 0.3596, + "num_tokens": 1110641136.0, + "reward": 1.8779296875, + "reward_std": 0.4632631838321686, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.88671875, + "rewards/format_reward/std": 0.3172462284564972, + "rewards/tag_count_reward/mean": 0.9404296875, + "rewards/tag_count_reward/std": 0.17864517867565155, + "step": 2041 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1914.0, + "completions/mean_length": 747.58203125, + "completions/mean_terminated_length": 711.0240478515625, + "completions/min_length": 142.0, + "completions/min_terminated_length": 142.0, + "epoch": 0.6971067679440129, + "grad_norm": 2.1707868576049805, + "kl": 4.99609375, + "learning_rate": 3.2919140136428727e-07, + "loss": 0.2871, + "num_tokens": 1111105162.0, + "reward": 1.8544921875, + "reward_std": 0.5191654562950134, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.9267578125, + "rewards/tag_count_reward/std": 0.192208394408226, + "step": 2042 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1916.0, + "completions/mean_length": 781.337890625, + "completions/mean_terminated_length": 756.1055908203125, + "completions/min_length": 135.0, + "completions/min_terminated_length": 135.0, + "epoch": 0.6974481522574038, + "grad_norm": 1.295634388923645, + "kl": 4.94140625, + "learning_rate": 3.287244276852223e-07, + "loss": 0.3002, + "num_tokens": 1111578871.0, + "reward": 1.88818359375, + "reward_std": 0.5161925554275513, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.86328125, + "rewards/format_reward/std": 0.3438861668109894, + "rewards/tag_count_reward/mean": 0.93505859375, + "rewards/tag_count_reward/std": 0.1798466295003891, + "step": 2043 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1916.0, + "completions/mean_length": 815.978515625, + "completions/mean_terminated_length": 778.7947387695312, + "completions/min_length": 183.0, + "completions/min_terminated_length": 183.0, + "epoch": 0.6977895365707946, + "grad_norm": 3.391709804534912, + "kl": 4.97265625, + "learning_rate": 3.282577680663604e-07, + "loss": 0.3362, + "num_tokens": 1112071068.0, + "reward": 1.82861328125, + "reward_std": 0.5299324989318848, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.857421875, + "rewards/format_reward/std": 0.3499840497970581, + "rewards/tag_count_reward/mean": 0.93212890625, + "rewards/tag_count_reward/std": 0.1867896467447281, + "step": 2044 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1923.0, + "completions/mean_length": 809.005859375, + "completions/mean_terminated_length": 763.8603515625, + "completions/min_length": 158.0, + "completions/min_terminated_length": 158.0, + "epoch": 0.6981309208841854, + "grad_norm": 4.1205902099609375, + "kl": 4.71875, + "learning_rate": 3.277914231700393e-07, + "loss": 0.3509, + "num_tokens": 1112566815.0, + "reward": 1.84375, + "reward_std": 0.5765846967697144, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.849609375, + "rewards/format_reward/std": 0.35780346393585205, + "rewards/tag_count_reward/mean": 0.923828125, + "rewards/tag_count_reward/std": 0.1967410445213318, + "step": 2045 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.060546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1949.0, + "completions/mean_length": 825.34375, + "completions/mean_terminated_length": 746.544677734375, + "completions/min_length": 117.0, + "completions/min_terminated_length": 117.0, + "epoch": 0.6984723051975762, + "grad_norm": 1.545742392539978, + "kl": 6.078125, + "learning_rate": 3.273253936581506e-07, + "loss": 0.3816, + "num_tokens": 1113069775.0, + "reward": 1.7470703125, + "reward_std": 0.5435930490493774, + "rewards/accuracy_reward/mean": 0.04435483738780022, + "rewards/accuracy_reward/std": 0.2060900777578354, + "rewards/format_reward/mean": 0.8046875, + "rewards/format_reward/std": 0.3968288004398346, + "rewards/tag_count_reward/mean": 0.8994140625, + "rewards/tag_count_reward/std": 0.22909599542617798, + "step": 2046 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2003.0, + "completions/mean_length": 800.427734375, + "completions/mean_terminated_length": 747.0692749023438, + "completions/min_length": 82.0, + "completions/min_terminated_length": 82.0, + "epoch": 0.698813689510967, + "grad_norm": 2.848963737487793, + "kl": 5.6953125, + "learning_rate": 3.2685968019213784e-07, + "loss": 0.4033, + "num_tokens": 1113552778.0, + "reward": 1.802734375, + "reward_std": 0.5537554025650024, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.919921875, + "rewards/tag_count_reward/std": 0.19890500605106354, + "step": 2047 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1913.0, + "completions/mean_length": 812.1015625, + "completions/mean_terminated_length": 759.2423706054688, + "completions/min_length": 28.0, + "completions/min_terminated_length": 28.0, + "epoch": 0.6991550738243578, + "grad_norm": 1.1179615259170532, + "kl": 7.1015625, + "learning_rate": 3.2639428343299623e-07, + "loss": 0.4567, + "num_tokens": 1114042814.0, + "reward": 1.76806640625, + "reward_std": 0.5800941586494446, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.81640625, + "rewards/format_reward/std": 0.3875311613082886, + "rewards/tag_count_reward/mean": 0.90869140625, + "rewards/tag_count_reward/std": 0.21235711872577667, + "step": 2048 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1941.0, + "completions/mean_length": 802.75390625, + "completions/mean_terminated_length": 752.1340942382812, + "completions/min_length": 96.0, + "completions/min_terminated_length": 96.0, + "epoch": 0.6994964581377485, + "grad_norm": 0.9869757890701294, + "kl": 6.78125, + "learning_rate": 3.259292040412711e-07, + "loss": 0.4029, + "num_tokens": 1114527936.0, + "reward": 1.72021484375, + "reward_std": 0.5575881600379944, + "rewards/accuracy_reward/mean": 0.025390625, + "rewards/accuracy_reward/std": 0.15746226906776428, + "rewards/format_reward/mean": 0.79296875, + "rewards/format_reward/std": 0.40557438135147095, + "rewards/tag_count_reward/mean": 0.90185546875, + "rewards/tag_count_reward/std": 0.21390387415885925, + "step": 2049 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1942.0, + "completions/mean_length": 769.087890625, + "completions/mean_terminated_length": 743.611572265625, + "completions/min_length": 145.0, + "completions/min_terminated_length": 145.0, + "epoch": 0.6998378424511393, + "grad_norm": 1.2725328207015991, + "kl": 5.4140625, + "learning_rate": 3.2546444267705786e-07, + "loss": 0.348, + "num_tokens": 1114997725.0, + "reward": 1.86083984375, + "reward_std": 0.5448645353317261, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.93115234375, + "rewards/tag_count_reward/std": 0.18110673129558563, + "step": 2050 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2021.0, + "completions/mean_length": 876.27734375, + "completions/mean_terminated_length": 813.5925903320312, + "completions/min_length": 27.0, + "completions/min_terminated_length": 27.0, + "epoch": 0.7001792267645301, + "grad_norm": 0.8814513087272644, + "kl": 7.859375, + "learning_rate": 3.250000000000001e-07, + "loss": 0.4884, + "num_tokens": 1115519995.0, + "reward": 1.79833984375, + "reward_std": 0.6143364906311035, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.8203125, + "rewards/format_reward/std": 0.38430243730545044, + "rewards/tag_count_reward/mean": 0.90966796875, + "rewards/tag_count_reward/std": 0.21277517080307007, + "step": 2051 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.056640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1994.0, + "completions/mean_length": 873.76953125, + "completions/mean_terminated_length": 803.26708984375, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "epoch": 0.700520611077921, + "grad_norm": 2.498657464981079, + "kl": 8.8671875, + "learning_rate": 3.245358766692891e-07, + "loss": 0.5218, + "num_tokens": 1116043653.0, + "reward": 1.7578125, + "reward_std": 0.5840620398521423, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.80859375, + "rewards/format_reward/std": 0.3937928080558777, + "rewards/tag_count_reward/mean": 0.90625, + "rewards/tag_count_reward/std": 0.21615298092365265, + "step": 2052 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1912.0, + "completions/mean_length": 889.03125, + "completions/mean_terminated_length": 824.5113525390625, + "completions/min_length": 185.0, + "completions/min_terminated_length": 185.0, + "epoch": 0.7008619953913118, + "grad_norm": 3.3309242725372314, + "kl": 7.984375, + "learning_rate": 3.2407207334366347e-07, + "loss": 0.4576, + "num_tokens": 1116575157.0, + "reward": 1.77685546875, + "reward_std": 0.6070467829704285, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.810546875, + "rewards/format_reward/std": 0.3922513723373413, + "rewards/tag_count_reward/mean": 0.90576171875, + "rewards/tag_count_reward/std": 0.2173515260219574, + "step": 2053 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 861.42578125, + "completions/mean_terminated_length": 797.9464721679688, + "completions/min_length": 188.0, + "completions/min_terminated_length": 188.0, + "epoch": 0.7012033797047026, + "grad_norm": 1.4844022989273071, + "kl": 7.8515625, + "learning_rate": 3.2360859068140666e-07, + "loss": 0.4569, + "num_tokens": 1117093103.0, + "reward": 1.75830078125, + "reward_std": 0.582671046257019, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.80859375, + "rewards/format_reward/std": 0.3937928080558777, + "rewards/tag_count_reward/mean": 0.90673828125, + "rewards/tag_count_reward/std": 0.21437686681747437, + "step": 2054 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1938.0, + "completions/mean_length": 811.24609375, + "completions/mean_terminated_length": 773.9194946289062, + "completions/min_length": 162.0, + "completions/min_terminated_length": 162.0, + "epoch": 0.7015447640180934, + "grad_norm": 0.7517781853675842, + "kl": 5.75, + "learning_rate": 3.2314542934034813e-07, + "loss": 0.3329, + "num_tokens": 1117587165.0, + "reward": 1.85107421875, + "reward_std": 0.5656074285507202, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.92333984375, + "rewards/tag_count_reward/std": 0.19561529159545898, + "step": 2055 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1975.0, + "completions/mean_length": 798.5546875, + "completions/mean_terminated_length": 763.4296875, + "completions/min_length": 141.0, + "completions/min_terminated_length": 141.0, + "epoch": 0.7018861483314842, + "grad_norm": 1.4017919301986694, + "kl": 5.7265625, + "learning_rate": 3.2268258997786015e-07, + "loss": 0.3736, + "num_tokens": 1118067785.0, + "reward": 1.904296875, + "reward_std": 0.5260640382766724, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.93359375, + "rewards/tag_count_reward/std": 0.18633443117141724, + "step": 2056 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2017.0, + "completions/mean_length": 862.255859375, + "completions/mean_terminated_length": 803.9405517578125, + "completions/min_length": 142.0, + "completions/min_terminated_length": 142.0, + "epoch": 0.7022275326448749, + "grad_norm": 2.1168081760406494, + "kl": 5.796875, + "learning_rate": 3.2222007325085885e-07, + "loss": 0.3847, + "num_tokens": 1118585004.0, + "reward": 1.80859375, + "reward_std": 0.5263746976852417, + "rewards/accuracy_reward/mean": 0.03125, + "rewards/accuracy_reward/std": 0.17416280508041382, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.921875, + "rewards/tag_count_reward/std": 0.2045232206583023, + "step": 2057 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1988.0, + "completions/mean_length": 765.3828125, + "completions/mean_terminated_length": 726.6719970703125, + "completions/min_length": 148.0, + "completions/min_terminated_length": 148.0, + "epoch": 0.7025689169582657, + "grad_norm": 0.7896247506141663, + "kl": 5.66796875, + "learning_rate": 3.217578798158022e-07, + "loss": 0.3404, + "num_tokens": 1119057200.0, + "reward": 1.88037109375, + "reward_std": 0.5610973834991455, + "rewards/accuracy_reward/mean": 0.1088709682226181, + "rewards/accuracy_reward/std": 0.3117917478084564, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.92919921875, + "rewards/tag_count_reward/std": 0.18635430932044983, + "step": 2058 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1924.0, + "completions/mean_length": 772.09375, + "completions/mean_terminated_length": 738.8536987304688, + "completions/min_length": 155.0, + "completions/min_terminated_length": 155.0, + "epoch": 0.7029103012716565, + "grad_norm": 1.0140058994293213, + "kl": 5.375, + "learning_rate": 3.2129601032868884e-07, + "loss": 0.326, + "num_tokens": 1119525584.0, + "reward": 1.93310546875, + "reward_std": 0.5191199779510498, + "rewards/accuracy_reward/mean": 0.134765625, + "rewards/accuracy_reward/std": 0.3418070077896118, + "rewards/format_reward/mean": 0.86328125, + "rewards/format_reward/std": 0.3438861668109894, + "rewards/tag_count_reward/mean": 0.93505859375, + "rewards/tag_count_reward/std": 0.17916527390480042, + "step": 2059 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1936.0, + "completions/mean_length": 865.33984375, + "completions/mean_terminated_length": 817.2642211914062, + "completions/min_length": 141.0, + "completions/min_terminated_length": 141.0, + "epoch": 0.7032516855850474, + "grad_norm": 1.2857820987701416, + "kl": 5.6796875, + "learning_rate": 3.2083446544505847e-07, + "loss": 0.3531, + "num_tokens": 1120048190.0, + "reward": 1.888671875, + "reward_std": 0.5044642686843872, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.880859375, + "rewards/format_reward/std": 0.32427072525024414, + "rewards/tag_count_reward/mean": 0.927734375, + "rewards/tag_count_reward/std": 0.19510231912136078, + "step": 2060 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1912.0, + "completions/mean_length": 844.173828125, + "completions/mean_terminated_length": 805.3406982421875, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "epoch": 0.7035930698984382, + "grad_norm": 1.0574839115142822, + "kl": 5.9765625, + "learning_rate": 3.203732458199893e-07, + "loss": 0.3572, + "num_tokens": 1120570471.0, + "reward": 1.828125, + "reward_std": 0.5073466300964355, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.849609375, + "rewards/format_reward/std": 0.35780346393585205, + "rewards/tag_count_reward/mean": 0.939453125, + "rewards/tag_count_reward/std": 0.1669810563325882, + "step": 2061 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1999.0, + "completions/mean_length": 777.623046875, + "completions/mean_terminated_length": 736.6431274414062, + "completions/min_length": 15.0, + "completions/min_terminated_length": 15.0, + "epoch": 0.703934454211829, + "grad_norm": 1.050209641456604, + "kl": 5.734375, + "learning_rate": 3.199123521080985e-07, + "loss": 0.3658, + "num_tokens": 1121042230.0, + "reward": 1.90234375, + "reward_std": 0.4538320302963257, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.884765625, + "rewards/format_reward/std": 0.3196168541908264, + "rewards/tag_count_reward/mean": 0.9375, + "rewards/tag_count_reward/std": 0.18037252128124237, + "step": 2062 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1984.0, + "completions/mean_length": 840.525390625, + "completions/mean_terminated_length": 816.4721069335938, + "completions/min_length": 183.0, + "completions/min_terminated_length": 183.0, + "epoch": 0.7042758385252198, + "grad_norm": 1.8714277744293213, + "kl": 6.3515625, + "learning_rate": 3.194517849635404e-07, + "loss": 0.3562, + "num_tokens": 1121552643.0, + "reward": 1.85205078125, + "reward_std": 0.5359213948249817, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.93212890625, + "rewards/tag_count_reward/std": 0.17249169945716858, + "step": 2063 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1841.0, + "completions/mean_length": 877.93359375, + "completions/mean_terminated_length": 825.3999633789062, + "completions/min_length": 127.0, + "completions/min_terminated_length": 127.0, + "epoch": 0.7046172228386106, + "grad_norm": 2.246628761291504, + "kl": 6.2265625, + "learning_rate": 3.1899154504000544e-07, + "loss": 0.3539, + "num_tokens": 1122087329.0, + "reward": 1.81689453125, + "reward_std": 0.5580817461013794, + "rewards/accuracy_reward/mean": 0.05645161122083664, + "rewards/accuracy_reward/std": 0.23102475702762604, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.92236328125, + "rewards/tag_count_reward/std": 0.19771909713745117, + "step": 2064 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2029.0, + "completions/mean_length": 822.955078125, + "completions/mean_terminated_length": 788.5160522460938, + "completions/min_length": 33.0, + "completions/min_terminated_length": 33.0, + "epoch": 0.7049586071520013, + "grad_norm": 1.6371833086013794, + "kl": 6.3984375, + "learning_rate": 3.185316329907204e-07, + "loss": 0.4093, + "num_tokens": 1122584394.0, + "reward": 1.89794921875, + "reward_std": 0.5305365324020386, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.93310546875, + "rewards/tag_count_reward/std": 0.18909280002117157, + "step": 2065 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1958.0, + "completions/mean_length": 839.705078125, + "completions/mean_terminated_length": 795.6781616210938, + "completions/min_length": 150.0, + "completions/min_terminated_length": 150.0, + "epoch": 0.7052999914653921, + "grad_norm": 1.2888261079788208, + "kl": 5.4921875, + "learning_rate": 3.1807204946844613e-07, + "loss": 0.355, + "num_tokens": 1123093411.0, + "reward": 1.888671875, + "reward_std": 0.4445631802082062, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.8984375, + "rewards/format_reward/std": 0.30236753821372986, + "rewards/tag_count_reward/mean": 0.943359375, + "rewards/tag_count_reward/std": 0.1719430834054947, + "step": 2066 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1993.0, + "completions/mean_length": 827.259765625, + "completions/mean_terminated_length": 795.4569091796875, + "completions/min_length": 194.0, + "completions/min_terminated_length": 194.0, + "epoch": 0.7056413757787829, + "grad_norm": 1.8677408695220947, + "kl": 3.96875, + "learning_rate": 3.176127951254775e-07, + "loss": 0.2618, + "num_tokens": 1123593800.0, + "reward": 1.890625, + "reward_std": 0.4381256699562073, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.90234375, + "rewards/format_reward/std": 0.29713961482048035, + "rewards/tag_count_reward/mean": 0.947265625, + "rewards/tag_count_reward/std": 0.1659708470106125, + "step": 2067 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 878.208984375, + "completions/mean_terminated_length": 825.687744140625, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "epoch": 0.7059827600921738, + "grad_norm": 1.1508092880249023, + "kl": 5.953125, + "learning_rate": 3.1715387061364187e-07, + "loss": 0.3571, + "num_tokens": 1124124851.0, + "reward": 1.83447265625, + "reward_std": 0.5666013956069946, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.92041015625, + "rewards/tag_count_reward/std": 0.20063117146492004, + "step": 2068 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2028.0, + "completions/mean_length": 855.765625, + "completions/mean_terminated_length": 804.7739868164062, + "completions/min_length": 206.0, + "completions/min_terminated_length": 206.0, + "epoch": 0.7063241444055646, + "grad_norm": 0.7897908687591553, + "kl": 5.796875, + "learning_rate": 3.1669527658429914e-07, + "loss": 0.3758, + "num_tokens": 1124644555.0, + "reward": 1.83154296875, + "reward_std": 0.5065468549728394, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.857421875, + "rewards/format_reward/std": 0.3499840497970581, + "rewards/tag_count_reward/mean": 0.92724609375, + "rewards/tag_count_reward/std": 0.19015607237815857, + "step": 2069 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1909.0, + "completions/mean_length": 840.5, + "completions/mean_terminated_length": 823.7623901367188, + "completions/min_length": 159.0, + "completions/min_terminated_length": 159.0, + "epoch": 0.7066655287189554, + "grad_norm": 1.5946892499923706, + "kl": 3.60546875, + "learning_rate": 3.162370136883389e-07, + "loss": 0.2418, + "num_tokens": 1125148203.0, + "reward": 1.9111328125, + "reward_std": 0.43999338150024414, + "rewards/accuracy_reward/mean": 0.060483869165182114, + "rewards/accuracy_reward/std": 0.2386218160390854, + "rewards/format_reward/mean": 0.90234375, + "rewards/format_reward/std": 0.29713961482048035, + "rewards/tag_count_reward/mean": 0.9501953125, + "rewards/tag_count_reward/std": 0.1593773365020752, + "step": 2070 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1939.0, + "completions/mean_length": 905.765625, + "completions/mean_terminated_length": 856.9124755859375, + "completions/min_length": 146.0, + "completions/min_terminated_length": 146.0, + "epoch": 0.7070069130323462, + "grad_norm": 1.0739637613296509, + "kl": 5.09765625, + "learning_rate": 3.157790825761818e-07, + "loss": 0.3311, + "num_tokens": 1125693235.0, + "reward": 1.8330078125, + "reward_std": 0.5031325817108154, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.9287109375, + "rewards/tag_count_reward/std": 0.18649590015411377, + "step": 2071 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1906.0, + "completions/mean_length": 867.619140625, + "completions/mean_terminated_length": 834.4357299804688, + "completions/min_length": 18.0, + "completions/min_terminated_length": 18.0, + "epoch": 0.707348297345737, + "grad_norm": 1.3727738857269287, + "kl": 5.53125, + "learning_rate": 3.1532148389777766e-07, + "loss": 0.3497, + "num_tokens": 1126212560.0, + "reward": 1.8515625, + "reward_std": 0.49354204535484314, + "rewards/accuracy_reward/mean": 0.037109375, + "rewards/accuracy_reward/std": 0.18921469151973724, + "rewards/format_reward/mean": 0.87890625, + "rewards/format_reward/std": 0.3265552520751953, + "rewards/tag_count_reward/mean": 0.935546875, + "rewards/tag_count_reward/std": 0.1783159226179123, + "step": 2072 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1906.0, + "completions/mean_length": 800.267578125, + "completions/mean_terminated_length": 744.2468872070312, + "completions/min_length": 7.0, + "completions/min_terminated_length": 7.0, + "epoch": 0.7076896816591277, + "grad_norm": 0.9355970025062561, + "kl": 5.55859375, + "learning_rate": 3.148642183026037e-07, + "loss": 0.3709, + "num_tokens": 1126692553.0, + "reward": 1.90087890625, + "reward_std": 0.4898828864097595, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.8828125, + "rewards/format_reward/std": 0.32195815443992615, + "rewards/tag_count_reward/mean": 0.93798828125, + "rewards/tag_count_reward/std": 0.17746655642986298, + "step": 2073 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2003.0, + "completions/mean_length": 878.673828125, + "completions/mean_terminated_length": 813.5773315429688, + "completions/min_length": 140.0, + "completions/min_terminated_length": 140.0, + "epoch": 0.7080310659725185, + "grad_norm": 0.9593584537506104, + "kl": 7.15625, + "learning_rate": 3.14407286439665e-07, + "loss": 0.4826, + "num_tokens": 1127221202.0, + "reward": 1.880859375, + "reward_std": 0.5013091564178467, + "rewards/accuracy_reward/mean": 0.07459677755832672, + "rewards/accuracy_reward/std": 0.263004869222641, + "rewards/format_reward/mean": 0.876953125, + "rewards/format_reward/std": 0.32881227135658264, + "rewards/tag_count_reward/mean": 0.931640625, + "rewards/tag_count_reward/std": 0.19400213658809662, + "step": 2074 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1947.0, + "completions/mean_length": 815.337890625, + "completions/mean_terminated_length": 757.3599243164062, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "epoch": 0.7083724502859093, + "grad_norm": 3.248323917388916, + "kl": 8.171875, + "learning_rate": 3.1395068895749275e-07, + "loss": 0.4901, + "num_tokens": 1127716383.0, + "reward": 1.89990234375, + "reward_std": 0.6105526089668274, + "rewards/accuracy_reward/mean": 0.138671875, + "rewards/accuracy_reward/std": 0.34594178199768066, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.91552734375, + "rewards/tag_count_reward/std": 0.20941190421581268, + "step": 2075 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2025.0, + "completions/mean_length": 888.046875, + "completions/mean_terminated_length": 825.9917602539062, + "completions/min_length": 109.0, + "completions/min_terminated_length": 109.0, + "epoch": 0.7087138345993002, + "grad_norm": 2.321079969406128, + "kl": 8.3671875, + "learning_rate": 3.134944265041436e-07, + "loss": 0.5019, + "num_tokens": 1128253431.0, + "reward": 1.79833984375, + "reward_std": 0.5561305284500122, + "rewards/accuracy_reward/mean": 0.0463709682226181, + "rewards/accuracy_reward/std": 0.21049949526786804, + "rewards/format_reward/mean": 0.8359375, + "rewards/format_reward/std": 0.37069445848464966, + "rewards/tag_count_reward/mean": 0.91748046875, + "rewards/tag_count_reward/std": 0.20428889989852905, + "step": 2076 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 880.76953125, + "completions/mean_terminated_length": 838.2388916015625, + "completions/min_length": 139.0, + "completions/min_terminated_length": 139.0, + "epoch": 0.709055218912691, + "grad_norm": 2.276273012161255, + "kl": 6.71875, + "learning_rate": 3.1303849972719834e-07, + "loss": 0.4276, + "num_tokens": 1128781393.0, + "reward": 1.86083984375, + "reward_std": 0.5237646102905273, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.857421875, + "rewards/format_reward/std": 0.3499840497970581, + "rewards/tag_count_reward/mean": 0.92724609375, + "rewards/tag_count_reward/std": 0.1869123876094818, + "step": 2077 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1934.0, + "completions/mean_length": 776.658203125, + "completions/mean_terminated_length": 740.9176635742188, + "completions/min_length": 139.0, + "completions/min_terminated_length": 139.0, + "epoch": 0.7093966032260818, + "grad_norm": 1.743714690208435, + "kl": 6.734375, + "learning_rate": 3.1258290927376187e-07, + "loss": 0.4157, + "num_tokens": 1129250066.0, + "reward": 1.88232421875, + "reward_std": 0.5077085494995117, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.93505859375, + "rewards/tag_count_reward/std": 0.18912817537784576, + "step": 2078 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1846.0, + "completions/mean_length": 836.994140625, + "completions/mean_terminated_length": 787.7662353515625, + "completions/min_length": 107.0, + "completions/min_terminated_length": 107.0, + "epoch": 0.7097379875394726, + "grad_norm": 1.4613759517669678, + "kl": 8.015625, + "learning_rate": 3.121276557904616e-07, + "loss": 0.4949, + "num_tokens": 1129758271.0, + "reward": 1.8251953125, + "reward_std": 0.5580568313598633, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.9208984375, + "rewards/tag_count_reward/std": 0.20534151792526245, + "step": 2079 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.064453125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2041.0, + "completions/mean_length": 941.126953125, + "completions/mean_terminated_length": 864.87060546875, + "completions/min_length": 228.0, + "completions/min_terminated_length": 228.0, + "epoch": 0.7100793718528634, + "grad_norm": 1.0365214347839355, + "kl": 7.7265625, + "learning_rate": 3.1167273992344646e-07, + "loss": 0.5085, + "num_tokens": 1130317376.0, + "reward": 1.75830078125, + "reward_std": 0.5934568643569946, + "rewards/accuracy_reward/mean": 0.04435483738780022, + "rewards/accuracy_reward/std": 0.2060900777578354, + "rewards/format_reward/mean": 0.822265625, + "rewards/format_reward/std": 0.3826628625392914, + "rewards/tag_count_reward/mean": 0.89306640625, + "rewards/tag_count_reward/std": 0.23650622367858887, + "step": 2080 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2033.0, + "completions/mean_length": 823.771484375, + "completions/mean_terminated_length": 794.3900146484375, + "completions/min_length": 20.0, + "completions/min_terminated_length": 20.0, + "epoch": 0.7104207561662541, + "grad_norm": 0.7497561573982239, + "kl": 5.75, + "learning_rate": 3.112181623183866e-07, + "loss": 0.3274, + "num_tokens": 1130817611.0, + "reward": 1.87353515625, + "reward_std": 0.5437583923339844, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.873046875, + "rewards/format_reward/std": 0.33324605226516724, + "rewards/tag_count_reward/mean": 0.92236328125, + "rewards/tag_count_reward/std": 0.19833672046661377, + "step": 2081 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 875.755859375, + "completions/mean_terminated_length": 828.1036376953125, + "completions/min_length": 120.0, + "completions/min_terminated_length": 120.0, + "epoch": 0.7107621404796449, + "grad_norm": 2.05602765083313, + "kl": 5.46875, + "learning_rate": 3.1076392362047117e-07, + "loss": 0.3592, + "num_tokens": 1131342606.0, + "reward": 1.8037109375, + "reward_std": 0.4788172245025635, + "rewards/accuracy_reward/mean": 0.01953125, + "rewards/accuracy_reward/std": 0.1385180652141571, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.9287109375, + "rewards/tag_count_reward/std": 0.18845312297344208, + "step": 2082 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1943.0, + "completions/mean_length": 845.427734375, + "completions/mean_terminated_length": 786.2847900390625, + "completions/min_length": 50.0, + "completions/min_terminated_length": 50.0, + "epoch": 0.7111035247930357, + "grad_norm": 0.9748495221138, + "kl": 6.421875, + "learning_rate": 3.1031002447440945e-07, + "loss": 0.4114, + "num_tokens": 1131848105.0, + "reward": 1.87890625, + "reward_std": 0.5175662040710449, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.873046875, + "rewards/format_reward/std": 0.33324605226516724, + "rewards/tag_count_reward/mean": 0.93359375, + "rewards/tag_count_reward/std": 0.18894177675247192, + "step": 2083 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1921.0, + "completions/mean_length": 829.20703125, + "completions/mean_terminated_length": 774.4856567382812, + "completions/min_length": 120.0, + "completions/min_terminated_length": 120.0, + "epoch": 0.7114449091064265, + "grad_norm": 1.3156884908676147, + "kl": 7.265625, + "learning_rate": 3.0985646552442794e-07, + "loss": 0.4885, + "num_tokens": 1132342147.0, + "reward": 1.7783203125, + "reward_std": 0.5539376139640808, + "rewards/accuracy_reward/mean": 0.033203125, + "rewards/accuracy_reward/std": 0.17934183776378632, + "rewards/format_reward/mean": 0.833984375, + "rewards/format_reward/std": 0.3724585771560669, + "rewards/tag_count_reward/mean": 0.9111328125, + "rewards/tag_count_reward/std": 0.2142503559589386, + "step": 2084 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.064453125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2014.0, + "completions/mean_length": 945.25, + "completions/mean_terminated_length": 869.2777099609375, + "completions/min_length": 231.0, + "completions/min_terminated_length": 231.0, + "epoch": 0.7117862934198174, + "grad_norm": 2.0443739891052246, + "kl": 7.4609375, + "learning_rate": 3.0940324741427103e-07, + "loss": 0.5197, + "num_tokens": 1132902563.0, + "reward": 1.7919921875, + "reward_std": 0.5862941145896912, + "rewards/accuracy_reward/mean": 0.037109375, + "rewards/accuracy_reward/std": 0.18921469151973724, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.9091796875, + "rewards/tag_count_reward/std": 0.21908392012119293, + "step": 2085 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 928.03515625, + "completions/mean_terminated_length": 863.2437744140625, + "completions/min_length": 128.0, + "completions/min_terminated_length": 128.0, + "epoch": 0.7121276777332082, + "grad_norm": 1.4917434453964233, + "kl": 6.5859375, + "learning_rate": 3.089503707871983e-07, + "loss": 0.3996, + "num_tokens": 1133452213.0, + "reward": 1.78466796875, + "reward_std": 0.5361355543136597, + "rewards/accuracy_reward/mean": 0.029296875, + "rewards/accuracy_reward/std": 0.16880230605602264, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.91552734375, + "rewards/tag_count_reward/std": 0.20647093653678894, + "step": 2086 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2025.0, + "completions/mean_length": 841.89453125, + "completions/mean_terminated_length": 790.3096313476562, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "epoch": 0.712469062046599, + "grad_norm": 1.0421783924102783, + "kl": 6.37890625, + "learning_rate": 3.084978362859859e-07, + "loss": 0.3985, + "num_tokens": 1133965023.0, + "reward": 1.830078125, + "reward_std": 0.5412536263465881, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.849609375, + "rewards/format_reward/std": 0.35780346393585205, + "rewards/tag_count_reward/mean": 0.919921875, + "rewards/tag_count_reward/std": 0.20316380262374878, + "step": 2087 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1933.0, + "completions/mean_length": 919.078125, + "completions/mean_terminated_length": 843.8167114257812, + "completions/min_length": 109.0, + "completions/min_terminated_length": 109.0, + "epoch": 0.7128104463599898, + "grad_norm": 1.837857723236084, + "kl": 8.21875, + "learning_rate": 3.080456445529237e-07, + "loss": 0.5007, + "num_tokens": 1134518903.0, + "reward": 1.783203125, + "reward_std": 0.6176773309707642, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.806640625, + "rewards/format_reward/std": 0.39531853795051575, + "rewards/tag_count_reward/mean": 0.90234375, + "rewards/tag_count_reward/std": 0.21948698163032532, + "step": 2088 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2009.0, + "completions/mean_length": 805.67578125, + "completions/mean_terminated_length": 752.5418090820312, + "completions/min_length": 18.0, + "completions/min_terminated_length": 18.0, + "epoch": 0.7131518306733805, + "grad_norm": 1.8352817296981812, + "kl": 6.84375, + "learning_rate": 3.075937962298147e-07, + "loss": 0.3996, + "num_tokens": 1135008753.0, + "reward": 1.83154296875, + "reward_std": 0.5388531684875488, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.849609375, + "rewards/format_reward/std": 0.35780346393585205, + "rewards/tag_count_reward/mean": 0.91943359375, + "rewards/tag_count_reward/std": 0.2074405401945114, + "step": 2089 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1961.0, + "completions/mean_length": 829.541015625, + "completions/mean_terminated_length": 802.7884521484375, + "completions/min_length": 196.0, + "completions/min_terminated_length": 196.0, + "epoch": 0.7134932149867713, + "grad_norm": 1.3277990818023682, + "kl": 5.046875, + "learning_rate": 3.0714229195797545e-07, + "loss": 0.3146, + "num_tokens": 1135506326.0, + "reward": 1.9111328125, + "reward_std": 0.4676492214202881, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.888671875, + "rewards/format_reward/std": 0.31484565138816833, + "rewards/tag_count_reward/mean": 0.9443359375, + "rewards/tag_count_reward/std": 0.16648533940315247, + "step": 2090 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1992.0, + "completions/mean_length": 790.603515625, + "completions/mean_terminated_length": 744.7874755859375, + "completions/min_length": 120.0, + "completions/min_terminated_length": 120.0, + "epoch": 0.7138345993001621, + "grad_norm": 1.3557393550872803, + "kl": 6.1640625, + "learning_rate": 3.066911323782333e-07, + "loss": 0.4, + "num_tokens": 1135986123.0, + "reward": 1.9697265625, + "reward_std": 0.562901496887207, + "rewards/accuracy_reward/mean": 0.16129031777381897, + "rewards/accuracy_reward/std": 0.3681698739528656, + "rewards/format_reward/mean": 0.880859375, + "rewards/format_reward/std": 0.32427072525024414, + "rewards/tag_count_reward/mean": 0.9326171875, + "rewards/tag_count_reward/std": 0.18466287851333618, + "step": 2091 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1888.0, + "completions/mean_length": 818.677734375, + "completions/mean_terminated_length": 776.4586181640625, + "completions/min_length": 143.0, + "completions/min_terminated_length": 143.0, + "epoch": 0.714175983613553, + "grad_norm": 2.0661044120788574, + "kl": 5.5703125, + "learning_rate": 3.062403181309271e-07, + "loss": 0.3337, + "num_tokens": 1136485350.0, + "reward": 1.84130859375, + "reward_std": 0.5803340673446655, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.841796875, + "rewards/format_reward/std": 0.36528825759887695, + "rewards/tag_count_reward/mean": 0.92529296875, + "rewards/tag_count_reward/std": 0.18809975683689117, + "step": 2092 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1999.0, + "completions/mean_length": 806.595703125, + "completions/mean_terminated_length": 763.961669921875, + "completions/min_length": 120.0, + "completions/min_terminated_length": 120.0, + "epoch": 0.7145173679269438, + "grad_norm": 0.8907326459884644, + "kl": 5.4296875, + "learning_rate": 3.057898498559049e-07, + "loss": 0.3671, + "num_tokens": 1136971111.0, + "reward": 1.89013671875, + "reward_std": 0.4978489279747009, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.93505859375, + "rewards/tag_count_reward/std": 0.18254666030406952, + "step": 2093 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2008.0, + "completions/mean_length": 818.623046875, + "completions/mean_terminated_length": 763.426513671875, + "completions/min_length": 102.0, + "completions/min_terminated_length": 102.0, + "epoch": 0.7148587522403346, + "grad_norm": 1.2013685703277588, + "kl": 6.78125, + "learning_rate": 3.053397281925244e-07, + "loss": 0.4381, + "num_tokens": 1137464518.0, + "reward": 1.86572265625, + "reward_std": 0.5542969107627869, + "rewards/accuracy_reward/mean": 0.087890625, + "rewards/accuracy_reward/std": 0.2834126651287079, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.92236328125, + "rewards/tag_count_reward/std": 0.19833672046661377, + "step": 2094 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 842.0859375, + "completions/mean_terminated_length": 780.1807250976562, + "completions/min_length": 52.0, + "completions/min_terminated_length": 52.0, + "epoch": 0.7152001365537254, + "grad_norm": 1.0693302154541016, + "kl": 5.84375, + "learning_rate": 3.0488995377965064e-07, + "loss": 0.373, + "num_tokens": 1137967250.0, + "reward": 1.84423828125, + "reward_std": 0.5321292281150818, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.92626953125, + "rewards/tag_count_reward/std": 0.1979798823595047, + "step": 2095 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 858.94921875, + "completions/mean_terminated_length": 815.6234741210938, + "completions/min_length": 54.0, + "completions/min_terminated_length": 54.0, + "epoch": 0.7155415208671162, + "grad_norm": 1.3301665782928467, + "kl": 5.2265625, + "learning_rate": 3.0444052725565614e-07, + "loss": 0.3372, + "num_tokens": 1138483112.0, + "reward": 1.85498046875, + "reward_std": 0.546319842338562, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.859375, + "rewards/format_reward/std": 0.3479743003845215, + "rewards/tag_count_reward/mean": 0.92724609375, + "rewards/tag_count_reward/std": 0.19079817831516266, + "step": 2096 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1911.0, + "completions/mean_length": 789.494140625, + "completions/mean_terminated_length": 743.6376953125, + "completions/min_length": 179.0, + "completions/min_terminated_length": 179.0, + "epoch": 0.7158829051805069, + "grad_norm": 1.5741279125213623, + "kl": 6.3671875, + "learning_rate": 3.0399144925841993e-07, + "loss": 0.3954, + "num_tokens": 1138958373.0, + "reward": 1.91943359375, + "reward_std": 0.517371416091919, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.87890625, + "rewards/format_reward/std": 0.3265552520751953, + "rewards/tag_count_reward/mean": 0.93505859375, + "rewards/tag_count_reward/std": 0.18783032894134521, + "step": 2097 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 748.955078125, + "completions/mean_terminated_length": 733.5513916015625, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "epoch": 0.7162242894938977, + "grad_norm": 1.4236983060836792, + "kl": 6.2109375, + "learning_rate": 3.0354272042532573e-07, + "loss": 0.369, + "num_tokens": 1139425758.0, + "reward": 1.81103515625, + "reward_std": 0.5365231037139893, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.849609375, + "rewards/format_reward/std": 0.35780346393585205, + "rewards/tag_count_reward/mean": 0.92236328125, + "rewards/tag_count_reward/std": 0.19397197663784027, + "step": 2098 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2015.0, + "completions/mean_length": 768.205078125, + "completions/mean_terminated_length": 732.2268676757812, + "completions/min_length": 71.0, + "completions/min_terminated_length": 71.0, + "epoch": 0.7165656738072885, + "grad_norm": 1.2227526903152466, + "kl": 6.6796875, + "learning_rate": 3.030943413932624e-07, + "loss": 0.4388, + "num_tokens": 1139892087.0, + "reward": 1.8447265625, + "reward_std": 0.5190199613571167, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.9267578125, + "rewards/tag_count_reward/std": 0.19347688555717468, + "step": 2099 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2030.0, + "completions/mean_length": 823.9765625, + "completions/mean_terminated_length": 771.6253051757812, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "epoch": 0.7169070581206793, + "grad_norm": 0.9703647494316101, + "kl": 7.328125, + "learning_rate": 3.0264631279862183e-07, + "loss": 0.4783, + "num_tokens": 1140382683.0, + "reward": 1.90625, + "reward_std": 0.586249589920044, + "rewards/accuracy_reward/mean": 0.13709677755832672, + "rewards/accuracy_reward/std": 0.34429675340652466, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.9296875, + "rewards/tag_count_reward/std": 0.18882036209106445, + "step": 2100 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2031.0, + "completions/mean_length": 883.41015625, + "completions/mean_terminated_length": 831.1224365234375, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "epoch": 0.7172484424340702, + "grad_norm": 1.013115406036377, + "kl": 5.859375, + "learning_rate": 3.021986352772985e-07, + "loss": 0.3741, + "num_tokens": 1140916173.0, + "reward": 1.8623046875, + "reward_std": 0.5495156049728394, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.9150390625, + "rewards/tag_count_reward/std": 0.21240492165088654, + "step": 2101 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1972.0, + "completions/mean_length": 744.212890625, + "completions/mean_terminated_length": 715.5868530273438, + "completions/min_length": 102.0, + "completions/min_terminated_length": 102.0, + "epoch": 0.717589826747461, + "grad_norm": 1.1057240962982178, + "kl": 4.38671875, + "learning_rate": 3.0175130946468894e-07, + "loss": 0.2734, + "num_tokens": 1141380762.0, + "reward": 1.9521484375, + "reward_std": 0.4787192642688751, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.90625, + "rewards/format_reward/std": 0.29176566004753113, + "rewards/tag_count_reward/mean": 0.9482421875, + "rewards/tag_count_reward/std": 0.1655413806438446, + "step": 2102 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1883.0, + "completions/mean_length": 824.912109375, + "completions/mean_terminated_length": 780.34619140625, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "epoch": 0.7179312110608518, + "grad_norm": 0.7997674942016602, + "kl": 5.7421875, + "learning_rate": 3.013043359956903e-07, + "loss": 0.3634, + "num_tokens": 1141880909.0, + "reward": 1.9013671875, + "reward_std": 0.5231031179428101, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.873046875, + "rewards/format_reward/std": 0.33324605226516724, + "rewards/tag_count_reward/mean": 0.9345703125, + "rewards/tag_count_reward/std": 0.18733429908752441, + "step": 2103 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1989.0, + "completions/mean_length": 857.919921875, + "completions/mean_terminated_length": 812.0547485351562, + "completions/min_length": 114.0, + "completions/min_terminated_length": 114.0, + "epoch": 0.7182725953742426, + "grad_norm": 0.9730769991874695, + "kl": 7.140625, + "learning_rate": 3.008577155046997e-07, + "loss": 0.458, + "num_tokens": 1142396996.0, + "reward": 1.845703125, + "reward_std": 0.544554591178894, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.921875, + "rewards/tag_count_reward/std": 0.19783563911914825, + "step": 2104 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1996.0, + "completions/mean_length": 805.833984375, + "completions/mean_terminated_length": 750.063232421875, + "completions/min_length": 142.0, + "completions/min_terminated_length": 142.0, + "epoch": 0.7186139796876334, + "grad_norm": 0.9937102198600769, + "kl": 6.703125, + "learning_rate": 3.0041144862561307e-07, + "loss": 0.4164, + "num_tokens": 1142889791.0, + "reward": 1.8515625, + "reward_std": 0.5155594348907471, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.86328125, + "rewards/format_reward/std": 0.3438861668109894, + "rewards/tag_count_reward/mean": 0.92578125, + "rewards/tag_count_reward/std": 0.19810594618320465, + "step": 2105 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1930.0, + "completions/mean_length": 829.451171875, + "completions/mean_terminated_length": 795.1947631835938, + "completions/min_length": 127.0, + "completions/min_terminated_length": 127.0, + "epoch": 0.7189553640010241, + "grad_norm": 0.8206349611282349, + "kl": 6.078125, + "learning_rate": 2.9996553599182487e-07, + "loss": 0.3906, + "num_tokens": 1143390134.0, + "reward": 1.86279296875, + "reward_std": 0.44854938983917236, + "rewards/accuracy_reward/mean": 0.04233871027827263, + "rewards/accuracy_reward/std": 0.2015640139579773, + "rewards/format_reward/mean": 0.884765625, + "rewards/format_reward/std": 0.3196168541908264, + "rewards/tag_count_reward/mean": 0.93701171875, + "rewards/tag_count_reward/std": 0.1871933490037918, + "step": 2106 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2000.0, + "completions/mean_length": 814.73046875, + "completions/mean_terminated_length": 772.3757934570312, + "completions/min_length": 130.0, + "completions/min_terminated_length": 130.0, + "epoch": 0.7192967483144149, + "grad_norm": 1.5673121213912964, + "kl": 5.7890625, + "learning_rate": 2.995199782362264e-07, + "loss": 0.372, + "num_tokens": 1143887468.0, + "reward": 1.87353515625, + "reward_std": 0.5153354406356812, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.93603515625, + "rewards/tag_count_reward/std": 0.17883430421352386, + "step": 2107 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1857.0, + "completions/mean_length": 869.26953125, + "completions/mean_terminated_length": 818.8554077148438, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "epoch": 0.7196381326278057, + "grad_norm": 1.3445322513580322, + "kl": 8.1953125, + "learning_rate": 2.9907477599120537e-07, + "loss": 0.5169, + "num_tokens": 1144407734.0, + "reward": 1.8125, + "reward_std": 0.5680813789367676, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.916015625, + "rewards/tag_count_reward/std": 0.20518557727336884, + "step": 2108 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1883.0, + "completions/mean_length": 797.939453125, + "completions/mean_terminated_length": 752.3906860351562, + "completions/min_length": 154.0, + "completions/min_terminated_length": 154.0, + "epoch": 0.7199795169411966, + "grad_norm": 1.546714425086975, + "kl": 5.421875, + "learning_rate": 2.98629929888645e-07, + "loss": 0.3528, + "num_tokens": 1144889079.0, + "reward": 1.8740234375, + "reward_std": 0.5286086797714233, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.9306640625, + "rewards/tag_count_reward/std": 0.189181849360466, + "step": 2109 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1986.0, + "completions/mean_length": 816.01171875, + "completions/mean_terminated_length": 752.7680053710938, + "completions/min_length": 167.0, + "completions/min_terminated_length": 167.0, + "epoch": 0.7203209012545874, + "grad_norm": 1.4555652141571045, + "kl": 8.5703125, + "learning_rate": 2.981854405599228e-07, + "loss": 0.5657, + "num_tokens": 1145389805.0, + "reward": 1.7900390625, + "reward_std": 0.5721191167831421, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.833984375, + "rewards/format_reward/std": 0.3724585771560669, + "rewards/tag_count_reward/mean": 0.9169921875, + "rewards/tag_count_reward/std": 0.20736047625541687, + "step": 2110 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1891.0, + "completions/mean_length": 737.791015625, + "completions/mean_terminated_length": 684.5304565429688, + "completions/min_length": 29.0, + "completions/min_terminated_length": 29.0, + "epoch": 0.7206622855679782, + "grad_norm": 16.444395065307617, + "kl": 6.9140625, + "learning_rate": 2.9774130863591035e-07, + "loss": 0.4781, + "num_tokens": 1145849234.0, + "reward": 1.81494140625, + "reward_std": 0.4680173397064209, + "rewards/accuracy_reward/mean": 0.03515625, + "rewards/accuracy_reward/std": 0.1843547374010086, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.92822265625, + "rewards/tag_count_reward/std": 0.18794220685958862, + "step": 2111 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2010.0, + "completions/mean_length": 804.78125, + "completions/mean_terminated_length": 754.243896484375, + "completions/min_length": 27.0, + "completions/min_terminated_length": 27.0, + "epoch": 0.721003669881369, + "grad_norm": 1.175093650817871, + "kl": 7.2265625, + "learning_rate": 2.9729753474697157e-07, + "loss": 0.4748, + "num_tokens": 1146337218.0, + "reward": 1.87353515625, + "reward_std": 0.5310930013656616, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.93212890625, + "rewards/tag_count_reward/std": 0.19259266555309296, + "step": 2112 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 774.974609375, + "completions/mean_terminated_length": 733.9092407226562, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "epoch": 0.7213450541947598, + "grad_norm": 1.9142338037490845, + "kl": 5.5703125, + "learning_rate": 2.9685411952296214e-07, + "loss": 0.3121, + "num_tokens": 1146808229.0, + "reward": 1.90380859375, + "reward_std": 0.543915867805481, + "rewards/accuracy_reward/mean": 0.1015625, + "rewards/accuracy_reward/std": 0.30236753821372986, + "rewards/format_reward/mean": 0.8671875, + "rewards/format_reward/std": 0.33970388770103455, + "rewards/tag_count_reward/mean": 0.93505859375, + "rewards/tag_count_reward/std": 0.19233450293540955, + "step": 2113 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1875.0, + "completions/mean_length": 832.79296875, + "completions/mean_terminated_length": 770.4107055664062, + "completions/min_length": 204.0, + "completions/min_terminated_length": 204.0, + "epoch": 0.7216864385081505, + "grad_norm": 1.2744873762130737, + "kl": 7.84375, + "learning_rate": 2.964110635932292e-07, + "loss": 0.5237, + "num_tokens": 1147305019.0, + "reward": 1.8408203125, + "reward_std": 0.5451600551605225, + "rewards/accuracy_reward/mean": 0.05645161122083664, + "rewards/accuracy_reward/std": 0.23102474212646484, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.9248046875, + "rewards/tag_count_reward/std": 0.20382823050022125, + "step": 2114 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1978.0, + "completions/mean_length": 821.94140625, + "completions/mean_terminated_length": 753.6866455078125, + "completions/min_length": 38.0, + "completions/min_terminated_length": 38.0, + "epoch": 0.7220278228215413, + "grad_norm": 3.1961309909820557, + "kl": 8.390625, + "learning_rate": 2.959683675866093e-07, + "loss": 0.5083, + "num_tokens": 1147805693.0, + "reward": 1.83544921875, + "reward_std": 0.5922591686248779, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.830078125, + "rewards/format_reward/std": 0.3759314715862274, + "rewards/tag_count_reward/mean": 0.91162109375, + "rewards/tag_count_reward/std": 0.21071286499500275, + "step": 2115 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1888.0, + "completions/mean_length": 759.109375, + "completions/mean_terminated_length": 730.8103637695312, + "completions/min_length": 159.0, + "completions/min_terminated_length": 159.0, + "epoch": 0.7223692071349321, + "grad_norm": 1.4142956733703613, + "kl": 5.3984375, + "learning_rate": 2.955260321314287e-07, + "loss": 0.3445, + "num_tokens": 1148266917.0, + "reward": 1.88916015625, + "reward_std": 0.47626230120658875, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.89453125, + "rewards/format_reward/std": 0.3074568510055542, + "rewards/tag_count_reward/mean": 0.94384765625, + "rewards/tag_count_reward/std": 0.1745731234550476, + "step": 2116 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1935.0, + "completions/mean_length": 713.673828125, + "completions/mean_terminated_length": 676.16259765625, + "completions/min_length": 139.0, + "completions/min_terminated_length": 139.0, + "epoch": 0.722710591448323, + "grad_norm": 1.2257810831069946, + "kl": 6.125, + "learning_rate": 2.9508405785550144e-07, + "loss": 0.4101, + "num_tokens": 1148715262.0, + "reward": 1.89404296875, + "reward_std": 0.49057528376579285, + "rewards/accuracy_reward/mean": 0.07258064299821854, + "rewards/accuracy_reward/std": 0.25970885157585144, + "rewards/format_reward/mean": 0.884765625, + "rewards/format_reward/std": 0.3196168541908264, + "rewards/tag_count_reward/mean": 0.93896484375, + "rewards/tag_count_reward/std": 0.17711623013019562, + "step": 2117 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1966.0, + "completions/mean_length": 745.078125, + "completions/mean_terminated_length": 729.6284790039062, + "completions/min_length": 101.0, + "completions/min_terminated_length": 101.0, + "epoch": 0.7230519757617138, + "grad_norm": 1.1388176679611206, + "kl": 4.765625, + "learning_rate": 2.946424453861294e-07, + "loss": 0.2873, + "num_tokens": 1149176006.0, + "reward": 1.90673828125, + "reward_std": 0.5317778587341309, + "rewards/accuracy_reward/mean": 0.099609375, + "rewards/accuracy_reward/std": 0.29977133870124817, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.93603515625, + "rewards/tag_count_reward/std": 0.1781490594148636, + "step": 2118 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 824.03125, + "completions/mean_terminated_length": 784.54833984375, + "completions/min_length": 227.0, + "completions/min_terminated_length": 227.0, + "epoch": 0.7233933600751046, + "grad_norm": 0.9671112895011902, + "kl": 6.1171875, + "learning_rate": 2.942011953501007e-07, + "loss": 0.3821, + "num_tokens": 1149684438.0, + "reward": 1.796875, + "reward_std": 0.5368913412094116, + "rewards/accuracy_reward/mean": 0.04032257944345474, + "rewards/accuracy_reward/std": 0.19691328704357147, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.9140625, + "rewards/tag_count_reward/std": 0.21601147949695587, + "step": 2119 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 788.46484375, + "completions/mean_terminated_length": 755.6513061523438, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "epoch": 0.7237347443884954, + "grad_norm": 1.0110492706298828, + "kl": 6.1875, + "learning_rate": 2.937603083736887e-07, + "loss": 0.4005, + "num_tokens": 1150163460.0, + "reward": 1.8330078125, + "reward_std": 0.5243314504623413, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.9208984375, + "rewards/tag_count_reward/std": 0.19744645059108734, + "step": 2120 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1970.0, + "completions/mean_length": 792.935546875, + "completions/mean_terminated_length": 744.56591796875, + "completions/min_length": 83.0, + "completions/min_terminated_length": 83.0, + "epoch": 0.7240761287018862, + "grad_norm": 1.0699777603149414, + "kl": 5.484375, + "learning_rate": 2.9331978508265225e-07, + "loss": 0.3312, + "num_tokens": 1150658115.0, + "reward": 1.8525390625, + "reward_std": 0.5569126009941101, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.9248046875, + "rewards/tag_count_reward/std": 0.18362505733966827, + "step": 2121 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1816.0, + "completions/mean_length": 799.392578125, + "completions/mean_terminated_length": 764.2911376953125, + "completions/min_length": 148.0, + "completions/min_terminated_length": 148.0, + "epoch": 0.7244175130152769, + "grad_norm": 2.0671730041503906, + "kl": 5.00390625, + "learning_rate": 2.9287962610223326e-07, + "loss": 0.3268, + "num_tokens": 1151145644.0, + "reward": 1.94677734375, + "reward_std": 0.5002008676528931, + "rewards/accuracy_reward/mean": 0.125, + "rewards/accuracy_reward/std": 0.3310423493385315, + "rewards/format_reward/mean": 0.880859375, + "rewards/format_reward/std": 0.32427072525024414, + "rewards/tag_count_reward/mean": 0.94091796875, + "rewards/tag_count_reward/std": 0.17777857184410095, + "step": 2122 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1828.0, + "completions/mean_length": 750.310546875, + "completions/mean_terminated_length": 705.7434692382812, + "completions/min_length": 98.0, + "completions/min_terminated_length": 98.0, + "epoch": 0.7247588973286677, + "grad_norm": 2.3090503215789795, + "kl": 5.078125, + "learning_rate": 2.924398320571573e-07, + "loss": 0.3454, + "num_tokens": 1151608779.0, + "reward": 1.8623046875, + "reward_std": 0.4841301143169403, + "rewards/accuracy_reward/mean": 0.04583333432674408, + "rewards/accuracy_reward/std": 0.20934167504310608, + "rewards/format_reward/mean": 0.87890625, + "rewards/format_reward/std": 0.3265552520751953, + "rewards/tag_count_reward/mean": 0.9404296875, + "rewards/tag_count_reward/std": 0.17308135330677032, + "step": 2123 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2008.0, + "completions/mean_length": 794.26171875, + "completions/mean_terminated_length": 761.5991821289062, + "completions/min_length": 182.0, + "completions/min_terminated_length": 182.0, + "epoch": 0.7251002816420585, + "grad_norm": 1.55347740650177, + "kl": 7.2578125, + "learning_rate": 2.9200040357163114e-07, + "loss": 0.4675, + "num_tokens": 1152091841.0, + "reward": 1.81689453125, + "reward_std": 0.5591763257980347, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.92041015625, + "rewards/tag_count_reward/std": 0.19693946838378906, + "step": 2124 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 748.294921875, + "completions/mean_terminated_length": 709.068359375, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "epoch": 0.7254416659554493, + "grad_norm": 3.6218671798706055, + "kl": 7.74609375, + "learning_rate": 2.915613412693435e-07, + "loss": 0.4376, + "num_tokens": 1152552744.0, + "reward": 1.8427734375, + "reward_std": 0.5550713539123535, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.830078125, + "rewards/format_reward/std": 0.3759314715862274, + "rewards/tag_count_reward/mean": 0.9150390625, + "rewards/tag_count_reward/std": 0.20478235185146332, + "step": 2125 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1860.0, + "completions/mean_length": 750.392578125, + "completions/mean_terminated_length": 721.9022216796875, + "completions/min_length": 69.0, + "completions/min_terminated_length": 69.0, + "epoch": 0.7257830502688402, + "grad_norm": 2.598684310913086, + "kl": 6.4765625, + "learning_rate": 2.911226457734628e-07, + "loss": 0.349, + "num_tokens": 1153013601.0, + "reward": 1.85595703125, + "reward_std": 0.5394971370697021, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.92822265625, + "rewards/tag_count_reward/std": 0.1892392933368683, + "step": 2126 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1960.0, + "completions/mean_length": 809.75390625, + "completions/mean_terminated_length": 754.1591796875, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "epoch": 0.726124434582231, + "grad_norm": 3.110694646835327, + "kl": 8.6328125, + "learning_rate": 2.90684317706637e-07, + "loss": 0.4948, + "num_tokens": 1153516003.0, + "reward": 1.75146484375, + "reward_std": 0.604299008846283, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.794921875, + "rewards/format_reward/std": 0.4041535556316376, + "rewards/tag_count_reward/mean": 0.90185546875, + "rewards/tag_count_reward/std": 0.21333131194114685, + "step": 2127 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 767.173828125, + "completions/mean_terminated_length": 728.51708984375, + "completions/min_length": 128.0, + "completions/min_terminated_length": 128.0, + "epoch": 0.7264658188956218, + "grad_norm": 1.4040441513061523, + "kl": 6.90625, + "learning_rate": 2.9024635769099287e-07, + "loss": 0.427, + "num_tokens": 1153991452.0, + "reward": 1.84033203125, + "reward_std": 0.5145344138145447, + "rewards/accuracy_reward/mean": 0.0463709682226181, + "rewards/accuracy_reward/std": 0.21049949526786804, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.93017578125, + "rewards/tag_count_reward/std": 0.18341873586177826, + "step": 2128 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2006.0, + "completions/mean_length": 761.314453125, + "completions/mean_terminated_length": 719.8084716796875, + "completions/min_length": 24.0, + "completions/min_terminated_length": 24.0, + "epoch": 0.7268072032090126, + "grad_norm": 2.685636043548584, + "kl": 7.5625, + "learning_rate": 2.8980876634813424e-07, + "loss": 0.4497, + "num_tokens": 1154456013.0, + "reward": 1.80126953125, + "reward_std": 0.5381090044975281, + "rewards/accuracy_reward/mean": 0.037109375, + "rewards/accuracy_reward/std": 0.18921469151973724, + "rewards/format_reward/mean": 0.841796875, + "rewards/format_reward/std": 0.36528825759887695, + "rewards/tag_count_reward/mean": 0.92236328125, + "rewards/tag_count_reward/std": 0.19397197663784027, + "step": 2129 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1924.0, + "completions/mean_length": 778.046875, + "completions/mean_terminated_length": 750.1636962890625, + "completions/min_length": 54.0, + "completions/min_terminated_length": 54.0, + "epoch": 0.7271485875224033, + "grad_norm": 1.834752082824707, + "kl": 5.53125, + "learning_rate": 2.8937154429914233e-07, + "loss": 0.3485, + "num_tokens": 1154933397.0, + "reward": 1.85888671875, + "reward_std": 0.5585477352142334, + "rewards/accuracy_reward/mean": 0.07258064299821854, + "rewards/accuracy_reward/std": 0.25970885157585144, + "rewards/format_reward/mean": 0.859375, + "rewards/format_reward/std": 0.3479743003845215, + "rewards/tag_count_reward/mean": 0.92919921875, + "rewards/tag_count_reward/std": 0.18635430932044983, + "step": 2130 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 764.798828125, + "completions/mean_terminated_length": 741.8389282226562, + "completions/min_length": 179.0, + "completions/min_terminated_length": 179.0, + "epoch": 0.7274899718357941, + "grad_norm": 1.4363107681274414, + "kl": 4.98046875, + "learning_rate": 2.889346921645737e-07, + "loss": 0.3136, + "num_tokens": 1155397854.0, + "reward": 1.90673828125, + "reward_std": 0.5185901522636414, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.880859375, + "rewards/format_reward/std": 0.32427072525024414, + "rewards/tag_count_reward/mean": 0.93603515625, + "rewards/tag_count_reward/std": 0.1828918159008026, + "step": 2131 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2033.0, + "completions/mean_length": 907.546875, + "completions/mean_terminated_length": 870.758056640625, + "completions/min_length": 211.0, + "completions/min_terminated_length": 211.0, + "epoch": 0.7278313561491849, + "grad_norm": 1.5458734035491943, + "kl": 4.8984375, + "learning_rate": 2.8849821056445983e-07, + "loss": 0.2985, + "num_tokens": 1155941974.0, + "reward": 1.8466796875, + "reward_std": 0.4652012586593628, + "rewards/accuracy_reward/mean": 0.03427419438958168, + "rewards/accuracy_reward/std": 0.18211629986763, + "rewards/format_reward/mean": 0.880859375, + "rewards/format_reward/std": 0.32427072525024414, + "rewards/tag_count_reward/mean": 0.9326171875, + "rewards/tag_count_reward/std": 0.1911715567111969, + "step": 2132 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1965.0, + "completions/mean_length": 848.89453125, + "completions/mean_terminated_length": 805.2024536132812, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "epoch": 0.7281727404625757, + "grad_norm": 0.8934493064880371, + "kl": 5.80859375, + "learning_rate": 2.880621001183069e-07, + "loss": 0.3515, + "num_tokens": 1156451184.0, + "reward": 1.81640625, + "reward_std": 0.48416393995285034, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.921875, + "rewards/tag_count_reward/std": 0.20392431318759918, + "step": 2133 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2015.0, + "completions/mean_length": 758.8828125, + "completions/mean_terminated_length": 730.578857421875, + "completions/min_length": 83.0, + "completions/min_terminated_length": 83.0, + "epoch": 0.7285141247759666, + "grad_norm": 2.6557486057281494, + "kl": 4.7890625, + "learning_rate": 2.8762636144509366e-07, + "loss": 0.2927, + "num_tokens": 1156919844.0, + "reward": 1.82080078125, + "reward_std": 0.5306501388549805, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.849609375, + "rewards/format_reward/std": 0.35780346393585205, + "rewards/tag_count_reward/mean": 0.92822265625, + "rewards/tag_count_reward/std": 0.18663610517978668, + "step": 2134 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1999.0, + "completions/mean_length": 812.04296875, + "completions/mean_terminated_length": 769.5960083007812, + "completions/min_length": 149.0, + "completions/min_terminated_length": 149.0, + "epoch": 0.7288555090893574, + "grad_norm": 1.5461405515670776, + "kl": 5.6796875, + "learning_rate": 2.871909951632716e-07, + "loss": 0.3617, + "num_tokens": 1157407946.0, + "reward": 1.8916015625, + "reward_std": 0.5402153730392456, + "rewards/accuracy_reward/mean": 0.1015625, + "rewards/accuracy_reward/std": 0.30236753821372986, + "rewards/format_reward/mean": 0.859375, + "rewards/format_reward/std": 0.3479743003845215, + "rewards/tag_count_reward/mean": 0.9306640625, + "rewards/tag_count_reward/std": 0.18526214361190796, + "step": 2135 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 834.62890625, + "completions/mean_terminated_length": 800.51806640625, + "completions/min_length": 191.0, + "completions/min_terminated_length": 191.0, + "epoch": 0.7291968934027482, + "grad_norm": 1.1518545150756836, + "kl": 5.6875, + "learning_rate": 2.867560018907634e-07, + "loss": 0.3536, + "num_tokens": 1157917660.0, + "reward": 1.8671875, + "reward_std": 0.4876805245876312, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.9296875, + "rewards/tag_count_reward/std": 0.19139383733272552, + "step": 2136 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 908.12890625, + "completions/mean_terminated_length": 849.614013671875, + "completions/min_length": 164.0, + "completions/min_terminated_length": 164.0, + "epoch": 0.729538277716139, + "grad_norm": 1.1337326765060425, + "kl": 7.015625, + "learning_rate": 2.863213822449629e-07, + "loss": 0.4198, + "num_tokens": 1158467358.0, + "reward": 1.80859375, + "reward_std": 0.5352450609207153, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.841796875, + "rewards/format_reward/std": 0.36528825759887695, + "rewards/tag_count_reward/mean": 0.919921875, + "rewards/tag_count_reward/std": 0.20436429977416992, + "step": 2137 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 842.193359375, + "completions/mean_terminated_length": 793.1768188476562, + "completions/min_length": 163.0, + "completions/min_terminated_length": 163.0, + "epoch": 0.7298796620295297, + "grad_norm": 2.4628489017486572, + "kl": 7.234375, + "learning_rate": 2.8588713684273247e-07, + "loss": 0.4458, + "num_tokens": 1158976961.0, + "reward": 1.85595703125, + "reward_std": 0.5720502734184265, + "rewards/accuracy_reward/mean": 0.099609375, + "rewards/accuracy_reward/std": 0.29977133870124817, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.91845703125, + "rewards/tag_count_reward/std": 0.19861595332622528, + "step": 2138 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1967.0, + "completions/mean_length": 819.2578125, + "completions/mean_terminated_length": 771.902587890625, + "completions/min_length": 167.0, + "completions/min_terminated_length": 167.0, + "epoch": 0.7302210463429205, + "grad_norm": 1.3563839197158813, + "kl": 5.94140625, + "learning_rate": 2.8545326630040436e-07, + "loss": 0.3266, + "num_tokens": 1159472389.0, + "reward": 1.83837890625, + "reward_std": 0.5455411076545715, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.92236328125, + "rewards/tag_count_reward/std": 0.19709952175617218, + "step": 2139 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1965.0, + "completions/mean_length": 793.595703125, + "completions/mean_terminated_length": 758.331298828125, + "completions/min_length": 151.0, + "completions/min_terminated_length": 151.0, + "epoch": 0.7305624306563113, + "grad_norm": 1.2304555177688599, + "kl": 5.65625, + "learning_rate": 2.850197712337786e-07, + "loss": 0.3686, + "num_tokens": 1159954070.0, + "reward": 1.9013671875, + "reward_std": 0.5251258611679077, + "rewards/accuracy_reward/mean": 0.103515625, + "rewards/accuracy_reward/std": 0.30492907762527466, + "rewards/format_reward/mean": 0.8671875, + "rewards/format_reward/std": 0.33970388770103455, + "rewards/tag_count_reward/mean": 0.9306640625, + "rewards/tag_count_reward/std": 0.18526214361190796, + "step": 2140 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 873.259765625, + "completions/mean_terminated_length": 830.4555053710938, + "completions/min_length": 192.0, + "completions/min_terminated_length": 192.0, + "epoch": 0.7309038149697021, + "grad_norm": 1.5168801546096802, + "kl": 6.171875, + "learning_rate": 2.84586652258122e-07, + "loss": 0.3865, + "num_tokens": 1160479995.0, + "reward": 1.81396484375, + "reward_std": 0.47107118368148804, + "rewards/accuracy_reward/mean": 0.02016128972172737, + "rewards/accuracy_reward/std": 0.14069372415542603, + "rewards/format_reward/mean": 0.857421875, + "rewards/format_reward/std": 0.3499840497970581, + "rewards/tag_count_reward/mean": 0.93701171875, + "rewards/tag_count_reward/std": 0.1791812628507614, + "step": 2141 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2017.0, + "completions/mean_length": 787.8828125, + "completions/mean_terminated_length": 739.3184204101562, + "completions/min_length": 69.0, + "completions/min_terminated_length": 69.0, + "epoch": 0.731245199283093, + "grad_norm": 1.4604676961898804, + "kl": 7.3671875, + "learning_rate": 2.841539099881678e-07, + "loss": 0.4537, + "num_tokens": 1160954559.0, + "reward": 1.83935546875, + "reward_std": 0.590590238571167, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.91357421875, + "rewards/tag_count_reward/std": 0.21210047602653503, + "step": 2142 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2030.0, + "completions/mean_length": 819.16015625, + "completions/mean_terminated_length": 756.0780639648438, + "completions/min_length": 128.0, + "completions/min_terminated_length": 128.0, + "epoch": 0.7315865835964838, + "grad_norm": 1.804317831993103, + "kl": 8.890625, + "learning_rate": 2.837215450381144e-07, + "loss": 0.5505, + "num_tokens": 1161449489.0, + "reward": 1.80712890625, + "reward_std": 0.583137571811676, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.826171875, + "rewards/format_reward/std": 0.3793322443962097, + "rewards/tag_count_reward/mean": 0.90478515625, + "rewards/tag_count_reward/std": 0.2224915623664856, + "step": 2143 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 824.85546875, + "completions/mean_terminated_length": 772.5418090820312, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "epoch": 0.7319279679098746, + "grad_norm": 1.139277458190918, + "kl": 6.7734375, + "learning_rate": 2.832895580216249e-07, + "loss": 0.4459, + "num_tokens": 1161948711.0, + "reward": 1.833984375, + "reward_std": 0.4912651777267456, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.8671875, + "rewards/format_reward/std": 0.33970388770103455, + "rewards/tag_count_reward/mean": 0.92578125, + "rewards/tag_count_reward/std": 0.19994954764842987, + "step": 2144 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2008.0, + "completions/mean_length": 856.619140625, + "completions/mean_terminated_length": 823.12646484375, + "completions/min_length": 52.0, + "completions/min_terminated_length": 52.0, + "epoch": 0.7322693522232654, + "grad_norm": 1.1441174745559692, + "kl": 6.625, + "learning_rate": 2.828579495518256e-07, + "loss": 0.3795, + "num_tokens": 1162463700.0, + "reward": 1.83984375, + "reward_std": 0.49727314710617065, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.921875, + "rewards/tag_count_reward/std": 0.19345958530902863, + "step": 2145 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 820.228515625, + "completions/mean_terminated_length": 772.9107055664062, + "completions/min_length": 171.0, + "completions/min_terminated_length": 171.0, + "epoch": 0.7326107365366561, + "grad_norm": 2.1744227409362793, + "kl": 4.58984375, + "learning_rate": 2.824267202413061e-07, + "loss": 0.2925, + "num_tokens": 1162964681.0, + "reward": 1.95166015625, + "reward_std": 0.5318840742111206, + "rewards/accuracy_reward/mean": 0.130859375, + "rewards/accuracy_reward/std": 0.33757632970809937, + "rewards/format_reward/mean": 0.884765625, + "rewards/format_reward/std": 0.3196168541908264, + "rewards/tag_count_reward/mean": 0.93603515625, + "rewards/tag_count_reward/std": 0.184224471449852, + "step": 2146 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1998.0, + "completions/mean_length": 852.888671875, + "completions/mean_terminated_length": 829.0817260742188, + "completions/min_length": 37.0, + "completions/min_terminated_length": 37.0, + "epoch": 0.7329521208500469, + "grad_norm": 1.288240909576416, + "kl": 5.5625, + "learning_rate": 2.8199587070211737e-07, + "loss": 0.2962, + "num_tokens": 1163483440.0, + "reward": 1.78369140625, + "reward_std": 0.5132990479469299, + "rewards/accuracy_reward/mean": 0.04233871027827263, + "rewards/accuracy_reward/std": 0.2015640139579773, + "rewards/format_reward/mean": 0.826171875, + "rewards/format_reward/std": 0.3793322443962097, + "rewards/tag_count_reward/mean": 0.91650390625, + "rewards/tag_count_reward/std": 0.20086915791034698, + "step": 2147 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 865.533203125, + "completions/mean_terminated_length": 799.7052001953125, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "epoch": 0.7332935051634377, + "grad_norm": 0.9720637202262878, + "kl": 6.9296875, + "learning_rate": 2.815654015457715e-07, + "loss": 0.4364, + "num_tokens": 1164014721.0, + "reward": 1.81396484375, + "reward_std": 0.5638447999954224, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.8359375, + "rewards/format_reward/std": 0.37069445848464966, + "rewards/tag_count_reward/mean": 0.91357421875, + "rewards/tag_count_reward/std": 0.20743593573570251, + "step": 2148 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2012.0, + "completions/mean_length": 849.125, + "completions/mean_terminated_length": 807.9515380859375, + "completions/min_length": 193.0, + "completions/min_terminated_length": 193.0, + "epoch": 0.7336348894768285, + "grad_norm": 1.410672664642334, + "kl": 7.0078125, + "learning_rate": 2.8113531338324104e-07, + "loss": 0.4254, + "num_tokens": 1164529473.0, + "reward": 1.8828125, + "reward_std": 0.5692057609558105, + "rewards/accuracy_reward/mean": 0.109375, + "rewards/accuracy_reward/std": 0.31241437792778015, + "rewards/format_reward/mean": 0.849609375, + "rewards/format_reward/std": 0.35780346393585205, + "rewards/tag_count_reward/mean": 0.923828125, + "rewards/tag_count_reward/std": 0.19234009087085724, + "step": 2149 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1940.0, + "completions/mean_length": 747.064453125, + "completions/mean_terminated_length": 713.17236328125, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "epoch": 0.7339762737902193, + "grad_norm": 1.658525824546814, + "kl": 5.96875, + "learning_rate": 2.807056068249569e-07, + "loss": 0.3843, + "num_tokens": 1164984818.0, + "reward": 1.8369140625, + "reward_std": 0.578458309173584, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.8359375, + "rewards/format_reward/std": 0.37069445848464966, + "rewards/tag_count_reward/mean": 0.9248046875, + "rewards/tag_count_reward/std": 0.18692579865455627, + "step": 2150 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1956.0, + "completions/mean_length": 812.7265625, + "completions/mean_terminated_length": 759.8941040039062, + "completions/min_length": 165.0, + "completions/min_terminated_length": 165.0, + "epoch": 0.7343176581036102, + "grad_norm": 2.316544771194458, + "kl": 7.13671875, + "learning_rate": 2.8027628248080944e-07, + "loss": 0.459, + "num_tokens": 1165473270.0, + "reward": 1.83935546875, + "reward_std": 0.5644776821136475, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.91357421875, + "rewards/tag_count_reward/std": 0.2068454772233963, + "step": 2151 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1841.0, + "completions/mean_length": 772.55859375, + "completions/mean_terminated_length": 741.9480590820312, + "completions/min_length": 78.0, + "completions/min_terminated_length": 78.0, + "epoch": 0.734659042417001, + "grad_norm": 1.7509030103683472, + "kl": 5.00390625, + "learning_rate": 2.7984734096014567e-07, + "loss": 0.3087, + "num_tokens": 1165943492.0, + "reward": 1.966796875, + "reward_std": 0.5468528270721436, + "rewards/accuracy_reward/mean": 0.162109375, + "rewards/accuracy_reward/std": 0.3689115643501282, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.935546875, + "rewards/tag_count_reward/std": 0.16988569498062134, + "step": 2152 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 791.978515625, + "completions/mean_terminated_length": 754.0703735351562, + "completions/min_length": 139.0, + "completions/min_terminated_length": 139.0, + "epoch": 0.7350004267303918, + "grad_norm": 1.330442190170288, + "kl": 6.3203125, + "learning_rate": 2.794187828717698e-07, + "loss": 0.3979, + "num_tokens": 1166420761.0, + "reward": 1.84375, + "reward_std": 0.5745775699615479, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.916015625, + "rewards/tag_count_reward/std": 0.20278719067573547, + "step": 2153 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2008.0, + "completions/mean_length": 856.05859375, + "completions/mean_terminated_length": 797.4384765625, + "completions/min_length": 130.0, + "completions/min_terminated_length": 130.0, + "epoch": 0.7353418110437825, + "grad_norm": 1.8320715427398682, + "kl": 6.828125, + "learning_rate": 2.789906088239419e-07, + "loss": 0.4784, + "num_tokens": 1166932583.0, + "reward": 1.82421875, + "reward_std": 0.5676283836364746, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.91796875, + "rewards/tag_count_reward/std": 0.19872237741947174, + "step": 2154 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1973.0, + "completions/mean_length": 809.400390625, + "completions/mean_terminated_length": 761.665283203125, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "epoch": 0.7356831953571733, + "grad_norm": 2.485974073410034, + "kl": 5.578125, + "learning_rate": 2.7856281942437635e-07, + "loss": 0.3566, + "num_tokens": 1167431924.0, + "reward": 1.87841796875, + "reward_std": 0.524060070514679, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.86328125, + "rewards/format_reward/std": 0.3438861668109894, + "rewards/tag_count_reward/mean": 0.92333984375, + "rewards/tag_count_reward/std": 0.1981005072593689, + "step": 2155 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 837.966796875, + "completions/mean_terminated_length": 801.4466552734375, + "completions/min_length": 126.0, + "completions/min_terminated_length": 126.0, + "epoch": 0.7360245796705641, + "grad_norm": 4.13815975189209, + "kl": 4.4921875, + "learning_rate": 2.781354152802422e-07, + "loss": 0.327, + "num_tokens": 1167937155.0, + "reward": 1.86767578125, + "reward_std": 0.539473295211792, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.93017578125, + "rewards/tag_count_reward/std": 0.18408437073230743, + "step": 2156 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 863.119140625, + "completions/mean_terminated_length": 819.9453735351562, + "completions/min_length": 146.0, + "completions/min_terminated_length": 146.0, + "epoch": 0.7363659639839549, + "grad_norm": 1.896857738494873, + "kl": 5.109375, + "learning_rate": 2.777083969981611e-07, + "loss": 0.3534, + "num_tokens": 1168448800.0, + "reward": 1.861328125, + "reward_std": 0.5006389617919922, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.8828125, + "rewards/format_reward/std": 0.32195815443992615, + "rewards/tag_count_reward/mean": 0.931640625, + "rewards/tag_count_reward/std": 0.1869385838508606, + "step": 2157 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.056640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2019.0, + "completions/mean_length": 872.470703125, + "completions/mean_terminated_length": 801.8902587890625, + "completions/min_length": 82.0, + "completions/min_terminated_length": 82.0, + "epoch": 0.7367073482973457, + "grad_norm": 2.221587896347046, + "kl": 7.5390625, + "learning_rate": 2.7728176518420786e-07, + "loss": 0.4785, + "num_tokens": 1168982625.0, + "reward": 1.77734375, + "reward_std": 0.5431791543960571, + "rewards/accuracy_reward/mean": 0.03515625, + "rewards/accuracy_reward/std": 0.1843547374010086, + "rewards/format_reward/mean": 0.828125, + "rewards/format_reward/std": 0.3776407241821289, + "rewards/tag_count_reward/mean": 0.9140625, + "rewards/tag_count_reward/std": 0.20497123897075653, + "step": 2158 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1928.0, + "completions/mean_length": 839.021484375, + "completions/mean_terminated_length": 800.0221557617188, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "epoch": 0.7370487326107366, + "grad_norm": 1.8725794553756714, + "kl": 5.33203125, + "learning_rate": 2.768555204439079e-07, + "loss": 0.3407, + "num_tokens": 1169487356.0, + "reward": 1.8466796875, + "reward_std": 0.47082775831222534, + "rewards/accuracy_reward/mean": 0.033203125, + "rewards/accuracy_reward/std": 0.17934183776378632, + "rewards/format_reward/mean": 0.876953125, + "rewards/format_reward/std": 0.32881227135658264, + "rewards/tag_count_reward/mean": 0.9365234375, + "rewards/tag_count_reward/std": 0.17521031200885773, + "step": 2159 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 801.978515625, + "completions/mean_terminated_length": 777.1574096679688, + "completions/min_length": 152.0, + "completions/min_terminated_length": 152.0, + "epoch": 0.7373901169241274, + "grad_norm": 1.1445845365524292, + "kl": 4.90234375, + "learning_rate": 2.764296633822379e-07, + "loss": 0.3161, + "num_tokens": 1169976049.0, + "reward": 1.87548828125, + "reward_std": 0.46409809589385986, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.888671875, + "rewards/format_reward/std": 0.31484565138816833, + "rewards/tag_count_reward/mean": 0.94384765625, + "rewards/tag_count_reward/std": 0.1659528613090515, + "step": 2160 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2022.0, + "completions/mean_length": 794.51171875, + "completions/mean_terminated_length": 772.0834350585938, + "completions/min_length": 110.0, + "completions/min_terminated_length": 110.0, + "epoch": 0.7377315012375182, + "grad_norm": 1.0854650735855103, + "kl": 4.8046875, + "learning_rate": 2.7600419460362416e-07, + "loss": 0.3206, + "num_tokens": 1170463351.0, + "reward": 1.92724609375, + "reward_std": 0.4725736081600189, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.896484375, + "rewards/format_reward/std": 0.30492907762527466, + "rewards/tag_count_reward/mean": 0.94482421875, + "rewards/tag_count_reward/std": 0.16028828918933868, + "step": 2161 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1735.0, + "completions/mean_length": 761.931640625, + "completions/mean_terminated_length": 741.5178833007812, + "completions/min_length": 109.0, + "completions/min_terminated_length": 109.0, + "epoch": 0.7380728855509089, + "grad_norm": 1.5088930130004883, + "kl": 4.546875, + "learning_rate": 2.7557911471194167e-07, + "loss": 0.2663, + "num_tokens": 1170925444.0, + "reward": 1.90625, + "reward_std": 0.4686610996723175, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.88671875, + "rewards/format_reward/std": 0.3172462284564972, + "rewards/tag_count_reward/mean": 0.947265625, + "rewards/tag_count_reward/std": 0.1659708470106125, + "step": 2162 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1983.0, + "completions/mean_length": 815.39453125, + "completions/mean_terminated_length": 770.4818115234375, + "completions/min_length": 129.0, + "completions/min_terminated_length": 129.0, + "epoch": 0.7384142698642997, + "grad_norm": 2.3725340366363525, + "kl": 8.6875, + "learning_rate": 2.7515442431051363e-07, + "loss": 0.5359, + "num_tokens": 1171425982.0, + "reward": 1.82861328125, + "reward_std": 0.5628782510757446, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.92041015625, + "rewards/tag_count_reward/std": 0.20722854137420654, + "step": 2163 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1950.0, + "completions/mean_length": 874.740234375, + "completions/mean_terminated_length": 831.9899291992188, + "completions/min_length": 168.0, + "completions/min_terminated_length": 168.0, + "epoch": 0.7387556541776905, + "grad_norm": 2.1587789058685303, + "kl": 7.765625, + "learning_rate": 2.747301240021101e-07, + "loss": 0.4554, + "num_tokens": 1171950025.0, + "reward": 1.8076171875, + "reward_std": 0.5351447463035583, + "rewards/accuracy_reward/mean": 0.03515625, + "rewards/accuracy_reward/std": 0.1843547374010086, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.9208984375, + "rewards/tag_count_reward/std": 0.1968260258436203, + "step": 2164 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1902.0, + "completions/mean_length": 862.8671875, + "completions/mean_terminated_length": 822.1657104492188, + "completions/min_length": 143.0, + "completions/min_terminated_length": 143.0, + "epoch": 0.7390970384910813, + "grad_norm": 2.766791582107544, + "kl": 6.2890625, + "learning_rate": 2.7430621438894816e-07, + "loss": 0.3801, + "num_tokens": 1172459973.0, + "reward": 1.8203125, + "reward_std": 0.45686471462249756, + "rewards/accuracy_reward/mean": 0.017578125, + "rewards/accuracy_reward/std": 0.13154059648513794, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.93359375, + "rewards/tag_count_reward/std": 0.1816815733909607, + "step": 2165 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1959.0, + "completions/mean_length": 846.95703125, + "completions/mean_terminated_length": 810.7081909179688, + "completions/min_length": 112.0, + "completions/min_terminated_length": 112.0, + "epoch": 0.7394384228044721, + "grad_norm": 2.988097667694092, + "kl": 6.1171875, + "learning_rate": 2.7388269607268967e-07, + "loss": 0.3347, + "num_tokens": 1172974895.0, + "reward": 1.833984375, + "reward_std": 0.4791252613067627, + "rewards/accuracy_reward/mean": 0.02734375, + "rewards/accuracy_reward/std": 0.16324250400066376, + "rewards/format_reward/mean": 0.875, + "rewards/format_reward/std": 0.3310423493385315, + "rewards/tag_count_reward/mean": 0.931640625, + "rewards/tag_count_reward/std": 0.19018182158470154, + "step": 2166 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 817.06640625, + "completions/mean_terminated_length": 761.7999877929688, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "epoch": 0.739779807117863, + "grad_norm": 1.7172340154647827, + "kl": 7.24609375, + "learning_rate": 2.734595696544416e-07, + "loss": 0.4631, + "num_tokens": 1173469329.0, + "reward": 1.81787109375, + "reward_std": 0.5089281797409058, + "rewards/accuracy_reward/mean": 0.029296875, + "rewards/accuracy_reward/std": 0.16880230605602264, + "rewards/format_reward/mean": 0.86328125, + "rewards/format_reward/std": 0.3438861668109894, + "rewards/tag_count_reward/mean": 0.92529296875, + "rewards/tag_count_reward/std": 0.20189879834651947, + "step": 2167 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 804.94140625, + "completions/mean_terminated_length": 772.55712890625, + "completions/min_length": 186.0, + "completions/min_terminated_length": 186.0, + "epoch": 0.7401211914312538, + "grad_norm": 0.8305309414863586, + "kl": 5.4453125, + "learning_rate": 2.730368357347548e-07, + "loss": 0.3683, + "num_tokens": 1173968115.0, + "reward": 1.89111328125, + "reward_std": 0.5046444535255432, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.875, + "rewards/format_reward/std": 0.3310423493385315, + "rewards/tag_count_reward/mean": 0.93798828125, + "rewards/tag_count_reward/std": 0.17257477343082428, + "step": 2168 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1933.0, + "completions/mean_length": 857.11328125, + "completions/mean_terminated_length": 798.5450439453125, + "completions/min_length": 119.0, + "completions/min_terminated_length": 119.0, + "epoch": 0.7404625757446446, + "grad_norm": 1.792198896408081, + "kl": 6.65625, + "learning_rate": 2.7261449491362197e-07, + "loss": 0.4595, + "num_tokens": 1174481533.0, + "reward": 1.849609375, + "reward_std": 0.49879589676856995, + "rewards/accuracy_reward/mean": 0.037109375, + "rewards/accuracy_reward/std": 0.18921469151973724, + "rewards/format_reward/mean": 0.87890625, + "rewards/format_reward/std": 0.3265552520751953, + "rewards/tag_count_reward/mean": 0.93359375, + "rewards/tag_count_reward/std": 0.19342006742954254, + "step": 2169 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1974.0, + "completions/mean_length": 832.763671875, + "completions/mean_terminated_length": 783.36376953125, + "completions/min_length": 146.0, + "completions/min_terminated_length": 146.0, + "epoch": 0.7408039600580353, + "grad_norm": 1.9688204526901245, + "kl": 4.9375, + "learning_rate": 2.721925477904794e-07, + "loss": 0.3188, + "num_tokens": 1174988260.0, + "reward": 1.88671875, + "reward_std": 0.5032538175582886, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.87890625, + "rewards/format_reward/std": 0.3265552520751953, + "rewards/tag_count_reward/mean": 0.935546875, + "rewards/tag_count_reward/std": 0.18636520206928253, + "step": 2170 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1981.0, + "completions/mean_length": 881.09765625, + "completions/mean_terminated_length": 818.6707763671875, + "completions/min_length": 190.0, + "completions/min_terminated_length": 190.0, + "epoch": 0.7411453443714261, + "grad_norm": 2.3532350063323975, + "kl": 6.265625, + "learning_rate": 2.717709949642034e-07, + "loss": 0.4217, + "num_tokens": 1175514406.0, + "reward": 1.8876953125, + "reward_std": 0.5550177097320557, + "rewards/accuracy_reward/mean": 0.1015625, + "rewards/accuracy_reward/std": 0.30236753821372986, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.9248046875, + "rewards/tag_count_reward/std": 0.20141369104385376, + "step": 2171 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2015.0, + "completions/mean_length": 792.400390625, + "completions/mean_terminated_length": 754.5050048828125, + "completions/min_length": 153.0, + "completions/min_terminated_length": 153.0, + "epoch": 0.7414867286848169, + "grad_norm": 2.590756893157959, + "kl": 5.26171875, + "learning_rate": 2.7134983703311136e-07, + "loss": 0.3631, + "num_tokens": 1175990595.0, + "reward": 1.91357421875, + "reward_std": 0.4403735101222992, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.91015625, + "rewards/format_reward/std": 0.2862374484539032, + "rewards/tag_count_reward/mean": 0.94482421875, + "rewards/tag_count_reward/std": 0.17971375584602356, + "step": 2172 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2023.0, + "completions/mean_length": 853.4296875, + "completions/mean_terminated_length": 809.90283203125, + "completions/min_length": 183.0, + "completions/min_terminated_length": 183.0, + "epoch": 0.7418281129982077, + "grad_norm": 1.6033027172088623, + "kl": 6.078125, + "learning_rate": 2.7092907459495973e-07, + "loss": 0.4233, + "num_tokens": 1176505327.0, + "reward": 1.8564453125, + "reward_std": 0.5025766491889954, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.8671875, + "rewards/format_reward/std": 0.33970388770103455, + "rewards/tag_count_reward/mean": 0.9326171875, + "rewards/tag_count_reward/std": 0.18859504163265228, + "step": 2173 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 832.955078125, + "completions/mean_terminated_length": 791.226318359375, + "completions/min_length": 216.0, + "completions/min_terminated_length": 216.0, + "epoch": 0.7421694973115985, + "grad_norm": 0.7020198106765747, + "kl": 7.4375, + "learning_rate": 2.7050870824694407e-07, + "loss": 0.4757, + "num_tokens": 1177006648.0, + "reward": 1.8623046875, + "reward_std": 0.5373298525810242, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.9287109375, + "rewards/tag_count_reward/std": 0.19546197354793549, + "step": 2174 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1935.0, + "completions/mean_length": 869.595703125, + "completions/mean_terminated_length": 829.1253051757812, + "completions/min_length": 155.0, + "completions/min_terminated_length": 155.0, + "epoch": 0.7425108816249893, + "grad_norm": 1.6543843746185303, + "kl": 6.453125, + "learning_rate": 2.700887385856974e-07, + "loss": 0.395, + "num_tokens": 1177535497.0, + "reward": 1.8525390625, + "reward_std": 0.5069622993469238, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.9326171875, + "rewards/tag_count_reward/std": 0.18729348480701447, + "step": 2175 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2024.0, + "completions/mean_length": 857.248046875, + "completions/mean_terminated_length": 826.2264404296875, + "completions/min_length": 142.0, + "completions/min_terminated_length": 142.0, + "epoch": 0.7428522659383802, + "grad_norm": 0.9050835967063904, + "kl": 5.4453125, + "learning_rate": 2.6966916620728966e-07, + "loss": 0.3058, + "num_tokens": 1178053832.0, + "reward": 1.85205078125, + "reward_std": 0.500805139541626, + "rewards/accuracy_reward/mean": 0.04032257944345474, + "rewards/accuracy_reward/std": 0.19691328704357147, + "rewards/format_reward/mean": 0.876953125, + "rewards/format_reward/std": 0.32881227135658264, + "rewards/tag_count_reward/mean": 0.93603515625, + "rewards/tag_count_reward/std": 0.18154938519001007, + "step": 2176 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2030.0, + "completions/mean_length": 887.3125, + "completions/mean_terminated_length": 840.1300659179688, + "completions/min_length": 193.0, + "completions/min_terminated_length": 193.0, + "epoch": 0.743193650251771, + "grad_norm": 1.8727673292160034, + "kl": 7.3828125, + "learning_rate": 2.6924999170722743e-07, + "loss": 0.4327, + "num_tokens": 1178597032.0, + "reward": 1.80029296875, + "reward_std": 0.4992356300354004, + "rewards/accuracy_reward/mean": 0.021484375, + "rewards/accuracy_reward/std": 0.14513419568538666, + "rewards/format_reward/mean": 0.859375, + "rewards/format_reward/std": 0.3479743003845215, + "rewards/tag_count_reward/mean": 0.91943359375, + "rewards/tag_count_reward/std": 0.20024023950099945, + "step": 2177 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2017.0, + "completions/mean_length": 865.021484375, + "completions/mean_terminated_length": 791.3921508789062, + "completions/min_length": 4.0, + "completions/min_terminated_length": 4.0, + "epoch": 0.7435350345651617, + "grad_norm": 2.3138346672058105, + "kl": 8.40625, + "learning_rate": 2.6883121568045197e-07, + "loss": 0.5271, + "num_tokens": 1179114051.0, + "reward": 1.81591796875, + "reward_std": 0.5568733811378479, + "rewards/accuracy_reward/mean": 0.04435483738780022, + "rewards/accuracy_reward/std": 0.2060900777578354, + "rewards/format_reward/mean": 0.859375, + "rewards/format_reward/std": 0.3479743003845215, + "rewards/tag_count_reward/mean": 0.91357421875, + "rewards/tag_count_reward/std": 0.22002561390399933, + "step": 2178 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1975.0, + "completions/mean_length": 855.201171875, + "completions/mean_terminated_length": 814.2363891601562, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "epoch": 0.7438764188785525, + "grad_norm": 3.3663816452026367, + "kl": 7.8125, + "learning_rate": 2.6841283872133954e-07, + "loss": 0.4395, + "num_tokens": 1179637770.0, + "reward": 1.80810546875, + "reward_std": 0.5312970280647278, + "rewards/accuracy_reward/mean": 0.058467742055654526, + "rewards/accuracy_reward/std": 0.23486268520355225, + "rewards/format_reward/mean": 0.833984375, + "rewards/format_reward/std": 0.3724585771560669, + "rewards/tag_count_reward/mean": 0.91748046875, + "rewards/tag_count_reward/std": 0.20005404949188232, + "step": 2179 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 879.1328125, + "completions/mean_terminated_length": 829.1405639648438, + "completions/min_length": 154.0, + "completions/min_terminated_length": 154.0, + "epoch": 0.7442178031919433, + "grad_norm": 2.449056625366211, + "kl": 8.90625, + "learning_rate": 2.6799486142369955e-07, + "loss": 0.5346, + "num_tokens": 1180159054.0, + "reward": 1.79052734375, + "reward_std": 0.5429601073265076, + "rewards/accuracy_reward/mean": 0.0234375, + "rewards/accuracy_reward/std": 0.15143637359142303, + "rewards/format_reward/mean": 0.849609375, + "rewards/format_reward/std": 0.35780346393585205, + "rewards/tag_count_reward/mean": 0.91748046875, + "rewards/tag_count_reward/std": 0.2030879557132721, + "step": 2180 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1995.0, + "completions/mean_length": 843.326171875, + "completions/mean_terminated_length": 809.4598388671875, + "completions/min_length": 131.0, + "completions/min_terminated_length": 131.0, + "epoch": 0.7445591875053341, + "grad_norm": 1.237591028213501, + "kl": 6.640625, + "learning_rate": 2.6757728438077414e-07, + "loss": 0.3992, + "num_tokens": 1180681429.0, + "reward": 1.8359375, + "reward_std": 0.5391745567321777, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.921875, + "rewards/tag_count_reward/std": 0.2015109360218048, + "step": 2181 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 863.49609375, + "completions/mean_terminated_length": 807.783203125, + "completions/min_length": 109.0, + "completions/min_terminated_length": 109.0, + "epoch": 0.7449005718187249, + "grad_norm": 1.4139468669891357, + "kl": 7.6953125, + "learning_rate": 2.6716010818523794e-07, + "loss": 0.4984, + "num_tokens": 1181201299.0, + "reward": 1.859375, + "reward_std": 0.5659919381141663, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.919921875, + "rewards/tag_count_reward/std": 0.20674438774585724, + "step": 2182 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1970.0, + "completions/mean_length": 831.076171875, + "completions/mean_terminated_length": 744.5167236328125, + "completions/min_length": 193.0, + "completions/min_terminated_length": 193.0, + "epoch": 0.7452419561321157, + "grad_norm": 0.7848081588745117, + "kl": 9.234375, + "learning_rate": 2.667433334291958e-07, + "loss": 0.6095, + "num_tokens": 1181699674.0, + "reward": 1.82080078125, + "reward_std": 0.5908867120742798, + "rewards/accuracy_reward/mean": 0.08669354766607285, + "rewards/accuracy_reward/std": 0.281669557094574, + "rewards/format_reward/mean": 0.830078125, + "rewards/format_reward/std": 0.3759314715862274, + "rewards/tag_count_reward/mean": 0.90673828125, + "rewards/tag_count_reward/std": 0.22056347131729126, + "step": 2183 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2017.0, + "completions/mean_length": 842.107421875, + "completions/mean_terminated_length": 780.2033081054688, + "completions/min_length": 162.0, + "completions/min_terminated_length": 162.0, + "epoch": 0.7455833404455066, + "grad_norm": 1.3622740507125854, + "kl": 6.9140625, + "learning_rate": 2.663269607041837e-07, + "loss": 0.4162, + "num_tokens": 1182215665.0, + "reward": 1.84716796875, + "reward_std": 0.5520458817481995, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.849609375, + "rewards/format_reward/std": 0.35780346393585205, + "rewards/tag_count_reward/mean": 0.91943359375, + "rewards/tag_count_reward/std": 0.2068500965833664, + "step": 2184 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.064453125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 892.083984375, + "completions/mean_terminated_length": 812.4488525390625, + "completions/min_length": 185.0, + "completions/min_terminated_length": 185.0, + "epoch": 0.7459247247588974, + "grad_norm": 1.429387092590332, + "kl": 8.0546875, + "learning_rate": 2.6591099060116625e-07, + "loss": 0.5478, + "num_tokens": 1182745356.0, + "reward": 1.8076171875, + "reward_std": 0.5870583653450012, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.830078125, + "rewards/format_reward/std": 0.3759314715862274, + "rewards/tag_count_reward/mean": 0.9130859375, + "rewards/tag_count_reward/std": 0.21045252680778503, + "step": 2185 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1993.0, + "completions/mean_length": 832.154296875, + "completions/mean_terminated_length": 769.7392578125, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "epoch": 0.7462661090722881, + "grad_norm": 2.683655261993408, + "kl": 7.5390625, + "learning_rate": 2.6549542371053714e-07, + "loss": 0.5169, + "num_tokens": 1183252475.0, + "reward": 1.7998046875, + "reward_std": 0.5724963545799255, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.8359375, + "rewards/format_reward/std": 0.37069445848464966, + "rewards/tag_count_reward/mean": 0.9150390625, + "rewards/tag_count_reward/std": 0.21008896827697754, + "step": 2186 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1906.0, + "completions/mean_length": 836.283203125, + "completions/mean_terminated_length": 794.668701171875, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "epoch": 0.7466074933856789, + "grad_norm": 1.7893295288085938, + "kl": 5.5703125, + "learning_rate": 2.650802606221175e-07, + "loss": 0.361, + "num_tokens": 1183757644.0, + "reward": 1.94970703125, + "reward_std": 0.5130065679550171, + "rewards/accuracy_reward/mean": 0.13671875, + "rewards/accuracy_reward/std": 0.3438861668109894, + "rewards/format_reward/mean": 0.876953125, + "rewards/format_reward/std": 0.32881227135658264, + "rewards/tag_count_reward/mean": 0.93603515625, + "rewards/tag_count_reward/std": 0.18751464784145355, + "step": 2187 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1989.0, + "completions/mean_length": 855.548828125, + "completions/mean_terminated_length": 809.59228515625, + "completions/min_length": 200.0, + "completions/min_terminated_length": 200.0, + "epoch": 0.7469488776990697, + "grad_norm": 1.3820817470550537, + "kl": 6.125, + "learning_rate": 2.6466550192515526e-07, + "loss": 0.4133, + "num_tokens": 1184272053.0, + "reward": 1.87451171875, + "reward_std": 0.5345189571380615, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.93505859375, + "rewards/tag_count_reward/std": 0.18454577028751373, + "step": 2188 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2016.0, + "completions/mean_length": 803.23828125, + "completions/mean_terminated_length": 773.364013671875, + "completions/min_length": 108.0, + "completions/min_terminated_length": 108.0, + "epoch": 0.7472902620124605, + "grad_norm": 1.103055715560913, + "kl": 5.5546875, + "learning_rate": 2.642511482083247e-07, + "loss": 0.3329, + "num_tokens": 1184758175.0, + "reward": 1.8681640625, + "reward_std": 0.5132533311843872, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.9345703125, + "rewards/tag_count_reward/std": 0.1847042590379715, + "step": 2189 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2018.0, + "completions/mean_length": 839.50390625, + "completions/mean_terminated_length": 792.9290161132812, + "completions/min_length": 172.0, + "completions/min_terminated_length": 172.0, + "epoch": 0.7476316463258513, + "grad_norm": 1.225283145904541, + "kl": 5.71875, + "learning_rate": 2.638372000597251e-07, + "loss": 0.3729, + "num_tokens": 1185262065.0, + "reward": 1.833984375, + "reward_std": 0.5245562195777893, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.921875, + "rewards/tag_count_reward/std": 0.20272120833396912, + "step": 2190 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1937.0, + "completions/mean_length": 891.169921875, + "completions/mean_terminated_length": 834.276611328125, + "completions/min_length": 72.0, + "completions/min_terminated_length": 72.0, + "epoch": 0.7479730306392421, + "grad_norm": 1.0358110666275024, + "kl": 6.359375, + "learning_rate": 2.634236580668802e-07, + "loss": 0.3795, + "num_tokens": 1185798568.0, + "reward": 1.82421875, + "reward_std": 0.5024953484535217, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.857421875, + "rewards/format_reward/std": 0.3499840497970581, + "rewards/tag_count_reward/mean": 0.923828125, + "rewards/tag_count_reward/std": 0.199825257062912, + "step": 2191 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1951.0, + "completions/mean_length": 894.794921875, + "completions/mean_terminated_length": 838.0798950195312, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "epoch": 0.748314414952633, + "grad_norm": 1.1366462707519531, + "kl": 6.828125, + "learning_rate": 2.630105228167369e-07, + "loss": 0.4337, + "num_tokens": 1186336559.0, + "reward": 1.8154296875, + "reward_std": 0.5645444989204407, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.9228515625, + "rewards/tag_count_reward/std": 0.19944952428340912, + "step": 2192 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2014.0, + "completions/mean_length": 816.83203125, + "completions/mean_terminated_length": 766.7845458984375, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "epoch": 0.7486557992660238, + "grad_norm": 1.255671739578247, + "kl": 6.5625, + "learning_rate": 2.625977948956656e-07, + "loss": 0.4054, + "num_tokens": 1186822057.0, + "reward": 1.81591796875, + "reward_std": 0.49111637473106384, + "rewards/accuracy_reward/mean": 0.0234375, + "rewards/accuracy_reward/std": 0.15143637359142303, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.92724609375, + "rewards/tag_count_reward/std": 0.19143815338611603, + "step": 2193 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2000.0, + "completions/mean_length": 828.25, + "completions/mean_terminated_length": 796.4729614257812, + "completions/min_length": 138.0, + "completions/min_terminated_length": 138.0, + "epoch": 0.7489971835794145, + "grad_norm": 1.0259623527526855, + "kl": 6.5234375, + "learning_rate": 2.621854748894578e-07, + "loss": 0.4083, + "num_tokens": 1187323561.0, + "reward": 1.83642578125, + "reward_std": 0.4819796085357666, + "rewards/accuracy_reward/mean": 0.037109375, + "rewards/accuracy_reward/std": 0.18921469151973724, + "rewards/format_reward/mean": 0.8671875, + "rewards/format_reward/std": 0.33970388770103455, + "rewards/tag_count_reward/mean": 0.93212890625, + "rewards/tag_count_reward/std": 0.1794423907995224, + "step": 2194 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2028.0, + "completions/mean_length": 761.302734375, + "completions/mean_terminated_length": 722.4688110351562, + "completions/min_length": 128.0, + "completions/min_terminated_length": 128.0, + "epoch": 0.7493385678928053, + "grad_norm": 1.4665205478668213, + "kl": 5.9921875, + "learning_rate": 2.6177356338332635e-07, + "loss": 0.3828, + "num_tokens": 1187795764.0, + "reward": 1.92724609375, + "reward_std": 0.5011767148971558, + "rewards/accuracy_reward/mean": 0.10282257944345474, + "rewards/accuracy_reward/std": 0.30403366684913635, + "rewards/format_reward/mean": 0.884765625, + "rewards/format_reward/std": 0.3196168541908264, + "rewards/tag_count_reward/mean": 0.94287109375, + "rewards/tag_count_reward/std": 0.17284587025642395, + "step": 2195 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 839.35546875, + "completions/mean_terminated_length": 805.3775024414062, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "epoch": 0.7496799522061961, + "grad_norm": 2.3356518745422363, + "kl": 6.1640625, + "learning_rate": 2.6136206096190445e-07, + "loss": 0.385, + "num_tokens": 1188306682.0, + "reward": 1.833984375, + "reward_std": 0.47078582644462585, + "rewards/accuracy_reward/mean": 0.037109375, + "rewards/accuracy_reward/std": 0.18921469151973724, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.927734375, + "rewards/tag_count_reward/std": 0.19572821259498596, + "step": 2196 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2033.0, + "completions/mean_length": 782.974609375, + "completions/mean_terminated_length": 731.55078125, + "completions/min_length": 182.0, + "completions/min_terminated_length": 182.0, + "epoch": 0.7500213365195869, + "grad_norm": 2.1885030269622803, + "kl": 6.37890625, + "learning_rate": 2.609509682092442e-07, + "loss": 0.3773, + "num_tokens": 1188789133.0, + "reward": 1.88134765625, + "reward_std": 0.5129489302635193, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.93212890625, + "rewards/tag_count_reward/std": 0.1834864467382431, + "step": 2197 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 819.435546875, + "completions/mean_terminated_length": 787.4288940429688, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "epoch": 0.7503627208329777, + "grad_norm": 1.824910044670105, + "kl": 5.765625, + "learning_rate": 2.6054028570881697e-07, + "loss": 0.3489, + "num_tokens": 1189283196.0, + "reward": 1.91357421875, + "reward_std": 0.5337316393852234, + "rewards/accuracy_reward/mean": 0.115234375, + "rewards/accuracy_reward/std": 0.3196168541908264, + "rewards/format_reward/mean": 0.859375, + "rewards/format_reward/std": 0.3479743003845215, + "rewards/tag_count_reward/mean": 0.93896484375, + "rewards/tag_count_reward/std": 0.1656993180513382, + "step": 2198 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 732.29296875, + "completions/mean_terminated_length": 714.0554809570312, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "epoch": 0.7507041051463685, + "grad_norm": 1.5997239351272583, + "kl": 5.65625, + "learning_rate": 2.6013001404351133e-07, + "loss": 0.3435, + "num_tokens": 1189730274.0, + "reward": 1.86376953125, + "reward_std": 0.5020423531532288, + "rewards/accuracy_reward/mean": 0.058467742055654526, + "rewards/accuracy_reward/std": 0.23486268520355225, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.93798828125, + "rewards/tag_count_reward/std": 0.1689939647912979, + "step": 2199 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 804.517578125, + "completions/mean_terminated_length": 761.8121337890625, + "completions/min_length": 171.0, + "completions/min_terminated_length": 171.0, + "epoch": 0.7510454894597594, + "grad_norm": 2.429795503616333, + "kl": 6.45703125, + "learning_rate": 2.5972015379563263e-07, + "loss": 0.417, + "num_tokens": 1190213915.0, + "reward": 1.86865234375, + "reward_std": 0.5221760272979736, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.93115234375, + "rewards/tag_count_reward/std": 0.18708612024784088, + "step": 2200 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1993.0, + "completions/mean_length": 844.150390625, + "completions/mean_terminated_length": 815.258056640625, + "completions/min_length": 160.0, + "completions/min_terminated_length": 160.0, + "epoch": 0.7513868737731502, + "grad_norm": 1.5156134366989136, + "kl": 4.828125, + "learning_rate": 2.5931070554690284e-07, + "loss": 0.2999, + "num_tokens": 1190722888.0, + "reward": 1.8701171875, + "reward_std": 0.49041420221328735, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.8828125, + "rewards/format_reward/std": 0.32195815443992615, + "rewards/tag_count_reward/mean": 0.9384765625, + "rewards/tag_count_reward/std": 0.1779806911945343, + "step": 2201 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 806.056640625, + "completions/mean_terminated_length": 768.5734252929688, + "completions/min_length": 61.0, + "completions/min_terminated_length": 61.0, + "epoch": 0.7517282580865409, + "grad_norm": 1.6273012161254883, + "kl": 4.71484375, + "learning_rate": 2.589016698784585e-07, + "loss": 0.3303, + "num_tokens": 1191204517.0, + "reward": 1.841796875, + "reward_std": 0.5300570726394653, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.9296875, + "rewards/tag_count_reward/std": 0.1808803677558899, + "step": 2202 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1959.0, + "completions/mean_length": 819.947265625, + "completions/mean_terminated_length": 785.4236450195312, + "completions/min_length": 162.0, + "completions/min_terminated_length": 162.0, + "epoch": 0.7520696423999317, + "grad_norm": 1.6526873111724854, + "kl": 4.7734375, + "learning_rate": 2.5849304737085143e-07, + "loss": 0.3211, + "num_tokens": 1191696170.0, + "reward": 1.91015625, + "reward_std": 0.5129385590553284, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.876953125, + "rewards/format_reward/std": 0.32881227135658264, + "rewards/tag_count_reward/mean": 0.935546875, + "rewards/tag_count_reward/std": 0.16988569498062134, + "step": 2203 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2025.0, + "completions/mean_length": 885.30078125, + "completions/mean_terminated_length": 818.0371704101562, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "epoch": 0.7524110267133225, + "grad_norm": 2.0095741748809814, + "kl": 6.8984375, + "learning_rate": 2.5808483860404605e-07, + "loss": 0.4544, + "num_tokens": 1192231028.0, + "reward": 1.80322265625, + "reward_std": 0.594826877117157, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.810546875, + "rewards/format_reward/std": 0.3922513723373413, + "rewards/tag_count_reward/mean": 0.91064453125, + "rewards/tag_count_reward/std": 0.20380185544490814, + "step": 2204 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1882.0, + "completions/mean_length": 828.533203125, + "completions/mean_terminated_length": 776.3768310546875, + "completions/min_length": 121.0, + "completions/min_terminated_length": 121.0, + "epoch": 0.7527524110267133, + "grad_norm": 1.1258745193481445, + "kl": 5.2890625, + "learning_rate": 2.576770441574204e-07, + "loss": 0.3411, + "num_tokens": 1192729221.0, + "reward": 1.7998046875, + "reward_std": 0.5032345056533813, + "rewards/accuracy_reward/mean": 0.03125, + "rewards/accuracy_reward/std": 0.17416280508041382, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.9248046875, + "rewards/tag_count_reward/std": 0.17890173196792603, + "step": 2205 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1961.0, + "completions/mean_length": 787.1328125, + "completions/mean_terminated_length": 746.4596557617188, + "completions/min_length": 165.0, + "completions/min_terminated_length": 165.0, + "epoch": 0.7530937953401041, + "grad_norm": 1.8775099515914917, + "kl": 5.5546875, + "learning_rate": 2.5726966460976406e-07, + "loss": 0.3404, + "num_tokens": 1193210649.0, + "reward": 1.8466796875, + "reward_std": 0.5258615016937256, + "rewards/accuracy_reward/mean": 0.07459677755832672, + "rewards/accuracy_reward/std": 0.263004869222641, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.9287109375, + "rewards/tag_count_reward/std": 0.17707644402980804, + "step": 2206 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2031.0, + "completions/mean_length": 878.25, + "completions/mean_terminated_length": 815.6707763671875, + "completions/min_length": 112.0, + "completions/min_terminated_length": 112.0, + "epoch": 0.7534351796534949, + "grad_norm": 1.806650996208191, + "kl": 5.80859375, + "learning_rate": 2.5686270053927743e-07, + "loss": 0.3776, + "num_tokens": 1193738217.0, + "reward": 1.86474609375, + "reward_std": 0.592460572719574, + "rewards/accuracy_reward/mean": 0.130859375, + "rewards/accuracy_reward/std": 0.33757632970809937, + "rewards/format_reward/mean": 0.826171875, + "rewards/format_reward/std": 0.3793322443962097, + "rewards/tag_count_reward/mean": 0.90771484375, + "rewards/tag_count_reward/std": 0.21251004934310913, + "step": 2207 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1911.0, + "completions/mean_length": 847.6328125, + "completions/mean_terminated_length": 816.3607177734375, + "completions/min_length": 177.0, + "completions/min_terminated_length": 177.0, + "epoch": 0.7537765639668857, + "grad_norm": 1.2657173871994019, + "kl": 5.484375, + "learning_rate": 2.5645615252357205e-07, + "loss": 0.3376, + "num_tokens": 1194255517.0, + "reward": 1.8271484375, + "reward_std": 0.5507446527481079, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.833984375, + "rewards/format_reward/std": 0.3724585771560669, + "rewards/tag_count_reward/mean": 0.9267578125, + "rewards/tag_count_reward/std": 0.17696848511695862, + "step": 2208 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2006.0, + "completions/mean_length": 813.611328125, + "completions/mean_terminated_length": 752.9036254882812, + "completions/min_length": 112.0, + "completions/min_terminated_length": 112.0, + "epoch": 0.7541179482802766, + "grad_norm": 2.1323258876800537, + "kl": 6.0, + "learning_rate": 2.560500211396681e-07, + "loss": 0.3812, + "num_tokens": 1194754086.0, + "reward": 1.85009765625, + "reward_std": 0.5097396373748779, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.849609375, + "rewards/format_reward/std": 0.35780346393585205, + "rewards/tag_count_reward/mean": 0.92236328125, + "rewards/tag_count_reward/std": 0.19460150599479675, + "step": 2209 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1969.0, + "completions/mean_length": 844.94140625, + "completions/mean_terminated_length": 790.926513671875, + "completions/min_length": 84.0, + "completions/min_terminated_length": 84.0, + "epoch": 0.7544593325936674, + "grad_norm": 1.9357986450195312, + "kl": 5.15234375, + "learning_rate": 2.556443069639951e-07, + "loss": 0.3005, + "num_tokens": 1195267832.0, + "reward": 1.83544921875, + "reward_std": 0.575616717338562, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.841796875, + "rewards/format_reward/std": 0.36528825759887695, + "rewards/tag_count_reward/mean": 0.92138671875, + "rewards/tag_count_reward/std": 0.20040719211101532, + "step": 2210 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 758.11328125, + "completions/mean_terminated_length": 735.0337524414062, + "completions/min_length": 114.0, + "completions/min_terminated_length": 114.0, + "epoch": 0.7548007169070581, + "grad_norm": 1.115371584892273, + "kl": 4.3828125, + "learning_rate": 2.5523901057238994e-07, + "loss": 0.2774, + "num_tokens": 1195727586.0, + "reward": 1.84912109375, + "reward_std": 0.45485416054725647, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.93310546875, + "rewards/tag_count_reward/std": 0.18048836290836334, + "step": 2211 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1870.0, + "completions/mean_length": 739.970703125, + "completions/mean_terminated_length": 700.492919921875, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "epoch": 0.7551421012204489, + "grad_norm": 1.701364517211914, + "kl": 5.03125, + "learning_rate": 2.5483413254009666e-07, + "loss": 0.3306, + "num_tokens": 1196186163.0, + "reward": 1.87939453125, + "reward_std": 0.5180987119674683, + "rewards/accuracy_reward/mean": 0.087890625, + "rewards/accuracy_reward/std": 0.2834126651287079, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.92626953125, + "rewards/tag_count_reward/std": 0.1979798823595047, + "step": 2212 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1973.0, + "completions/mean_length": 877.142578125, + "completions/mean_terminated_length": 817.0369873046875, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "epoch": 0.7554834855338397, + "grad_norm": 1.9199590682983398, + "kl": 6.8203125, + "learning_rate": 2.544296734417658e-07, + "loss": 0.4529, + "num_tokens": 1196708844.0, + "reward": 1.82861328125, + "reward_std": 0.5699664354324341, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.91845703125, + "rewards/tag_count_reward/std": 0.20587307214736938, + "step": 2213 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1985.0, + "completions/mean_length": 882.943359375, + "completions/mean_terminated_length": 850.1907348632812, + "completions/min_length": 196.0, + "completions/min_terminated_length": 196.0, + "epoch": 0.7558248698472305, + "grad_norm": 1.002043604850769, + "kl": 6.078125, + "learning_rate": 2.540256338514528e-07, + "loss": 0.3203, + "num_tokens": 1197239423.0, + "reward": 1.83203125, + "reward_std": 0.6202792525291443, + "rewards/accuracy_reward/mean": 0.1015625, + "rewards/accuracy_reward/std": 0.30236753821372986, + "rewards/format_reward/mean": 0.818359375, + "rewards/format_reward/std": 0.38592514395713806, + "rewards/tag_count_reward/mean": 0.912109375, + "rewards/tag_count_reward/std": 0.20413975417613983, + "step": 2214 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1975.0, + "completions/mean_length": 854.2890625, + "completions/mean_terminated_length": 810.7935180664062, + "completions/min_length": 169.0, + "completions/min_terminated_length": 169.0, + "epoch": 0.7561662541606213, + "grad_norm": 1.092342495918274, + "kl": 6.5390625, + "learning_rate": 2.536220143426182e-07, + "loss": 0.3901, + "num_tokens": 1197746675.0, + "reward": 1.86572265625, + "reward_std": 0.5874383449554443, + "rewards/accuracy_reward/mean": 0.107421875, + "rewards/accuracy_reward/std": 0.30995169281959534, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.91845703125, + "rewards/tag_count_reward/std": 0.2022770643234253, + "step": 2215 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1829.0, + "completions/mean_length": 835.908203125, + "completions/mean_terminated_length": 791.742919921875, + "completions/min_length": 166.0, + "completions/min_terminated_length": 166.0, + "epoch": 0.7565076384740121, + "grad_norm": 1.3190197944641113, + "kl": 7.03125, + "learning_rate": 2.532188154881258e-07, + "loss": 0.4324, + "num_tokens": 1198247268.0, + "reward": 1.87646484375, + "reward_std": 0.5922967195510864, + "rewards/accuracy_reward/mean": 0.10080645233392715, + "rewards/accuracy_reward/std": 0.30137622356414795, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.92724609375, + "rewards/tag_count_reward/std": 0.1933453381061554, + "step": 2216 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 853.01953125, + "completions/mean_terminated_length": 783.8883666992188, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "epoch": 0.756849022787403, + "grad_norm": 4.725752353668213, + "kl": 8.8203125, + "learning_rate": 2.528160378602431e-07, + "loss": 0.5049, + "num_tokens": 1198756238.0, + "reward": 1.82861328125, + "reward_std": 0.5383328795433044, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.841796875, + "rewards/format_reward/std": 0.36528825759887695, + "rewards/tag_count_reward/mean": 0.91455078125, + "rewards/tag_count_reward/std": 0.21307136118412018, + "step": 2217 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1977.0, + "completions/mean_length": 858.107421875, + "completions/mean_terminated_length": 807.2159423828125, + "completions/min_length": 54.0, + "completions/min_terminated_length": 54.0, + "epoch": 0.7571904071007938, + "grad_norm": 5.096704483032227, + "kl": 8.921875, + "learning_rate": 2.5241368203063875e-07, + "loss": 0.4461, + "num_tokens": 1199267301.0, + "reward": 1.8154296875, + "reward_std": 0.590232253074646, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.818359375, + "rewards/format_reward/std": 0.38592514395713806, + "rewards/tag_count_reward/mean": 0.9072265625, + "rewards/tag_count_reward/std": 0.22159916162490845, + "step": 2218 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 881.2734375, + "completions/mean_terminated_length": 818.85595703125, + "completions/min_length": 3.0, + "completions/min_terminated_length": 3.0, + "epoch": 0.7575317914141845, + "grad_norm": 2.777573585510254, + "kl": 7.6875, + "learning_rate": 2.5201174857038344e-07, + "loss": 0.4052, + "num_tokens": 1199792113.0, + "reward": 1.75732421875, + "reward_std": 0.6155025959014893, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.810546875, + "rewards/format_reward/std": 0.3922513723373413, + "rewards/tag_count_reward/mean": 0.90185546875, + "rewards/tag_count_reward/std": 0.22775058448314667, + "step": 2219 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2006.0, + "completions/mean_length": 844.798828125, + "completions/mean_terminated_length": 800.95751953125, + "completions/min_length": 112.0, + "completions/min_terminated_length": 112.0, + "epoch": 0.7578731757275753, + "grad_norm": 2.8424341678619385, + "kl": 8.015625, + "learning_rate": 2.516102380499483e-07, + "loss": 0.4617, + "num_tokens": 1200305802.0, + "reward": 1.79638671875, + "reward_std": 0.5715229511260986, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.828125, + "rewards/format_reward/std": 0.3776407241821289, + "rewards/tag_count_reward/mean": 0.91552734375, + "rewards/tag_count_reward/std": 0.2052827626466751, + "step": 2220 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1943.0, + "completions/mean_length": 841.0390625, + "completions/mean_terminated_length": 791.9755859375, + "completions/min_length": 197.0, + "completions/min_terminated_length": 197.0, + "epoch": 0.7582145600409661, + "grad_norm": 1.5034443140029907, + "kl": 7.1796875, + "learning_rate": 2.51209151039204e-07, + "loss": 0.4459, + "num_tokens": 1200812878.0, + "reward": 1.90966796875, + "reward_std": 0.5487245321273804, + "rewards/accuracy_reward/mean": 0.1171875, + "rewards/accuracy_reward/std": 0.32195815443992615, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.92724609375, + "rewards/tag_count_reward/std": 0.196482852101326, + "step": 2221 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.056640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1952.0, + "completions/mean_length": 868.2265625, + "completions/mean_terminated_length": 797.3912963867188, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "epoch": 0.7585559443543569, + "grad_norm": 1.365991473197937, + "kl": 7.8203125, + "learning_rate": 2.5080848810742027e-07, + "loss": 0.4995, + "num_tokens": 1201334722.0, + "reward": 1.83056640625, + "reward_std": 0.5755613446235657, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.91259765625, + "rewards/tag_count_reward/std": 0.2162717580795288, + "step": 2222 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.068359375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1850.0, + "completions/mean_length": 929.384765625, + "completions/mean_terminated_length": 847.3060302734375, + "completions/min_length": 115.0, + "completions/min_terminated_length": 115.0, + "epoch": 0.7588973286677477, + "grad_norm": 1.0183286666870117, + "kl": 7.46875, + "learning_rate": 2.504082498232648e-07, + "loss": 0.4397, + "num_tokens": 1201896583.0, + "reward": 1.78076171875, + "reward_std": 0.5611814856529236, + "rewards/accuracy_reward/mean": 0.037109375, + "rewards/accuracy_reward/std": 0.18921469151973724, + "rewards/format_reward/mean": 0.830078125, + "rewards/format_reward/std": 0.3759314715862274, + "rewards/tag_count_reward/mean": 0.91357421875, + "rewards/tag_count_reward/std": 0.21210047602653503, + "step": 2223 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2001.0, + "completions/mean_length": 826.6640625, + "completions/mean_terminated_length": 797.35205078125, + "completions/min_length": 165.0, + "completions/min_terminated_length": 165.0, + "epoch": 0.7592387129811385, + "grad_norm": 1.4680759906768799, + "kl": 6.6015625, + "learning_rate": 2.5000843675480264e-07, + "loss": 0.4407, + "num_tokens": 1202390939.0, + "reward": 1.87548828125, + "reward_std": 0.5191822052001953, + "rewards/accuracy_reward/mean": 0.07056451588869095, + "rewards/accuracy_reward/std": 0.25635457038879395, + "rewards/format_reward/mean": 0.873046875, + "rewards/format_reward/std": 0.33324605226516724, + "rewards/tag_count_reward/mean": 0.93408203125, + "rewards/tag_count_reward/std": 0.18617989122867584, + "step": 2224 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2006.0, + "completions/mean_length": 865.591796875, + "completions/mean_terminated_length": 822.5081176757812, + "completions/min_length": 170.0, + "completions/min_terminated_length": 170.0, + "epoch": 0.7595800972945294, + "grad_norm": 1.8334529399871826, + "kl": 6.265625, + "learning_rate": 2.4960904946949513e-07, + "loss": 0.4026, + "num_tokens": 1202907738.0, + "reward": 1.81298828125, + "reward_std": 0.5341126322746277, + "rewards/accuracy_reward/mean": 0.037109375, + "rewards/accuracy_reward/std": 0.18921469151973724, + "rewards/format_reward/mean": 0.857421875, + "rewards/format_reward/std": 0.3499840497970581, + "rewards/tag_count_reward/mean": 0.91845703125, + "rewards/tag_count_reward/std": 0.2117307335138321, + "step": 2225 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 836.8984375, + "completions/mean_terminated_length": 795.3051147460938, + "completions/min_length": 172.0, + "completions/min_terminated_length": 172.0, + "epoch": 0.7599214816079202, + "grad_norm": 3.2000937461853027, + "kl": 6.890625, + "learning_rate": 2.492100885341997e-07, + "loss": 0.4656, + "num_tokens": 1203409974.0, + "reward": 1.82177734375, + "reward_std": 0.586864173412323, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.91748046875, + "rewards/tag_count_reward/std": 0.2066698521375656, + "step": 2226 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 892.263671875, + "completions/mean_terminated_length": 837.9038696289062, + "completions/min_length": 126.0, + "completions/min_terminated_length": 126.0, + "epoch": 0.7602628659213109, + "grad_norm": 0.8177643418312073, + "kl": 6.3125, + "learning_rate": 2.4881155451516844e-07, + "loss": 0.4118, + "num_tokens": 1203950845.0, + "reward": 1.90087890625, + "reward_std": 0.5213083624839783, + "rewards/accuracy_reward/mean": 0.107421875, + "rewards/accuracy_reward/std": 0.30995169281959534, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.92431640625, + "rewards/tag_count_reward/std": 0.20633205771446228, + "step": 2227 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1995.0, + "completions/mean_length": 844.009765625, + "completions/mean_terminated_length": 787.38037109375, + "completions/min_length": 157.0, + "completions/min_terminated_length": 157.0, + "epoch": 0.7606042502347017, + "grad_norm": 2.746137857437134, + "kl": 5.3671875, + "learning_rate": 2.484134479780473e-07, + "loss": 0.3835, + "num_tokens": 1204455746.0, + "reward": 1.83837890625, + "reward_std": 0.4725128412246704, + "rewards/accuracy_reward/mean": 0.03515625, + "rewards/accuracy_reward/std": 0.1843547374010086, + "rewards/format_reward/mean": 0.873046875, + "rewards/format_reward/std": 0.33324605226516724, + "rewards/tag_count_reward/mean": 0.93017578125, + "rewards/tag_count_reward/std": 0.19754503667354584, + "step": 2228 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1887.0, + "completions/mean_length": 879.423828125, + "completions/mean_terminated_length": 796.3033447265625, + "completions/min_length": 222.0, + "completions/min_terminated_length": 222.0, + "epoch": 0.7609456345480925, + "grad_norm": 1.2507325410842896, + "kl": 7.0546875, + "learning_rate": 2.48015769487876e-07, + "loss": 0.448, + "num_tokens": 1204982187.0, + "reward": 1.7958984375, + "reward_std": 0.5250375866889954, + "rewards/accuracy_reward/mean": 0.04233871027827263, + "rewards/accuracy_reward/std": 0.2015640139579773, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.9150390625, + "rewards/tag_count_reward/std": 0.21008896827697754, + "step": 2229 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2012.0, + "completions/mean_length": 856.7421875, + "completions/mean_terminated_length": 798.1557006835938, + "completions/min_length": 142.0, + "completions/min_terminated_length": 142.0, + "epoch": 0.7612870188614833, + "grad_norm": 1.2659684419631958, + "kl": 7.09375, + "learning_rate": 2.476185196090862e-07, + "loss": 0.4265, + "num_tokens": 1205496663.0, + "reward": 1.810546875, + "reward_std": 0.5451551675796509, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.9140625, + "rewards/tag_count_reward/std": 0.2096906155347824, + "step": 2230 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1946.0, + "completions/mean_length": 799.755859375, + "completions/mean_terminated_length": 762.0824584960938, + "completions/min_length": 156.0, + "completions/min_terminated_length": 156.0, + "epoch": 0.7616284031748741, + "grad_norm": 1.7775336503982544, + "kl": 6.5, + "learning_rate": 2.472216989055015e-07, + "loss": 0.4011, + "num_tokens": 1205978778.0, + "reward": 1.81396484375, + "reward_std": 0.576111912727356, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.91748046875, + "rewards/tag_count_reward/std": 0.21019071340560913, + "step": 2231 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 905.30859375, + "completions/mean_terminated_length": 844.1769409179688, + "completions/min_length": 184.0, + "completions/min_terminated_length": 184.0, + "epoch": 0.7619697874882649, + "grad_norm": 2.275806188583374, + "kl": 8.7109375, + "learning_rate": 2.468253079403362e-07, + "loss": 0.5015, + "num_tokens": 1206514760.0, + "reward": 1.7783203125, + "reward_std": 0.6185402870178223, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.806640625, + "rewards/format_reward/std": 0.39531853795051575, + "rewards/tag_count_reward/mean": 0.8974609375, + "rewards/tag_count_reward/std": 0.23141992092132568, + "step": 2232 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2014.0, + "completions/mean_length": 831.134765625, + "completions/mean_terminated_length": 768.6673583984375, + "completions/min_length": 127.0, + "completions/min_terminated_length": 127.0, + "epoch": 0.7623111718016558, + "grad_norm": 1.531739592552185, + "kl": 8.0078125, + "learning_rate": 2.464293472761948e-07, + "loss": 0.5286, + "num_tokens": 1207030653.0, + "reward": 1.82177734375, + "reward_std": 0.5514932870864868, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.8359375, + "rewards/format_reward/std": 0.37069445848464966, + "rewards/tag_count_reward/mean": 0.91943359375, + "rewards/tag_count_reward/std": 0.20447123050689697, + "step": 2233 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1955.0, + "completions/mean_length": 854.140625, + "completions/mean_terminated_length": 813.139404296875, + "completions/min_length": 89.0, + "completions/min_terminated_length": 89.0, + "epoch": 0.7626525561150466, + "grad_norm": 1.9341306686401367, + "kl": 6.859375, + "learning_rate": 2.460338174750713e-07, + "loss": 0.3934, + "num_tokens": 1207549829.0, + "reward": 1.77490234375, + "reward_std": 0.5592837333679199, + "rewards/accuracy_reward/mean": 0.02734375, + "rewards/accuracy_reward/std": 0.16324250400066376, + "rewards/format_reward/mean": 0.8359375, + "rewards/format_reward/std": 0.37069445848464966, + "rewards/tag_count_reward/mean": 0.91162109375, + "rewards/tag_count_reward/std": 0.2089642435312271, + "step": 2234 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1849.0, + "completions/mean_length": 813.28125, + "completions/mean_terminated_length": 752.5573120117188, + "completions/min_length": 76.0, + "completions/min_terminated_length": 76.0, + "epoch": 0.7629939404284373, + "grad_norm": 1.682078242301941, + "kl": 7.671875, + "learning_rate": 2.4563871909834755e-07, + "loss": 0.4935, + "num_tokens": 1208044581.0, + "reward": 1.83203125, + "reward_std": 0.5297613739967346, + "rewards/accuracy_reward/mean": 0.058467742055654526, + "rewards/accuracy_reward/std": 0.23486268520355225, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.923828125, + "rewards/tag_count_reward/std": 0.1992122381925583, + "step": 2235 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 843.78125, + "completions/mean_terminated_length": 804.9354858398438, + "completions/min_length": 128.0, + "completions/min_terminated_length": 128.0, + "epoch": 0.7633353247418281, + "grad_norm": 1.6867430210113525, + "kl": 7.328125, + "learning_rate": 2.4524405270679386e-07, + "loss": 0.4435, + "num_tokens": 1208562085.0, + "reward": 1.82177734375, + "reward_std": 0.5510420203208923, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.24230584502220154, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.91748046875, + "rewards/tag_count_reward/std": 0.2060771882534027, + "step": 2236 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2024.0, + "completions/mean_length": 865.666015625, + "completions/mean_terminated_length": 817.6036376953125, + "completions/min_length": 108.0, + "completions/min_terminated_length": 108.0, + "epoch": 0.7636767090552189, + "grad_norm": 2.1145455837249756, + "kl": 7.265625, + "learning_rate": 2.4484981886056647e-07, + "loss": 0.4384, + "num_tokens": 1209085882.0, + "reward": 1.81201171875, + "reward_std": 0.5959441661834717, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.81640625, + "rewards/format_reward/std": 0.3875311613082886, + "rewards/tag_count_reward/mean": 0.91357421875, + "rewards/tag_count_reward/std": 0.2068454772233963, + "step": 2237 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1937.0, + "completions/mean_length": 848.09375, + "completions/mean_terminated_length": 794.2203979492188, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "epoch": 0.7640180933686097, + "grad_norm": 1.2139153480529785, + "kl": 5.515625, + "learning_rate": 2.444560181192087e-07, + "loss": 0.3383, + "num_tokens": 1209603162.0, + "reward": 1.88232421875, + "reward_std": 0.5427642464637756, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.92529296875, + "rewards/tag_count_reward/std": 0.19449345767498016, + "step": 2238 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2028.0, + "completions/mean_length": 869.572265625, + "completions/mean_terminated_length": 829.10107421875, + "completions/min_length": 174.0, + "completions/min_terminated_length": 174.0, + "epoch": 0.7643594776820005, + "grad_norm": 2.9974546432495117, + "kl": 5.22265625, + "learning_rate": 2.4406265104164814e-07, + "loss": 0.3595, + "num_tokens": 1210122351.0, + "reward": 1.87060546875, + "reward_std": 0.5087319016456604, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.93115234375, + "rewards/tag_count_reward/std": 0.18968312442302704, + "step": 2239 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 812.63671875, + "completions/mean_terminated_length": 775.3521118164062, + "completions/min_length": 159.0, + "completions/min_terminated_length": 159.0, + "epoch": 0.7647008619953913, + "grad_norm": 2.801384925842285, + "kl": 4.4375, + "learning_rate": 2.4366971818619785e-07, + "loss": 0.3232, + "num_tokens": 1210618165.0, + "reward": 1.943359375, + "reward_std": 0.500016450881958, + "rewards/accuracy_reward/mean": 0.099609375, + "rewards/accuracy_reward/std": 0.29977133870124817, + "rewards/format_reward/mean": 0.8984375, + "rewards/format_reward/std": 0.30236753821372986, + "rewards/tag_count_reward/mean": 0.9453125, + "rewards/tag_count_reward/std": 0.17884030938148499, + "step": 2240 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1958.0, + "completions/mean_length": 897.6953125, + "completions/mean_terminated_length": 826.099609375, + "completions/min_length": 165.0, + "completions/min_terminated_length": 165.0, + "epoch": 0.7650422463087821, + "grad_norm": 1.5173786878585815, + "kl": 6.7734375, + "learning_rate": 2.4327722011055407e-07, + "loss": 0.4403, + "num_tokens": 1211150841.0, + "reward": 1.77783203125, + "reward_std": 0.5543250441551208, + "rewards/accuracy_reward/mean": 0.03515625, + "rewards/accuracy_reward/std": 0.1843547374010086, + "rewards/format_reward/mean": 0.8359375, + "rewards/format_reward/std": 0.37069445848464966, + "rewards/tag_count_reward/mean": 0.90673828125, + "rewards/tag_count_reward/std": 0.2249559760093689, + "step": 2241 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 839.140625, + "completions/mean_terminated_length": 782.2821655273438, + "completions/min_length": 181.0, + "completions/min_terminated_length": 181.0, + "epoch": 0.765383630622173, + "grad_norm": 0.8445493578910828, + "kl": 6.60546875, + "learning_rate": 2.428851573717961e-07, + "loss": 0.4365, + "num_tokens": 1211663329.0, + "reward": 1.86767578125, + "reward_std": 0.5464333295822144, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.92041015625, + "rewards/tag_count_reward/std": 0.19940820336341858, + "step": 2242 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1965.0, + "completions/mean_length": 874.228515625, + "completions/mean_terminated_length": 828.9918823242188, + "completions/min_length": 210.0, + "completions/min_terminated_length": 210.0, + "epoch": 0.7657250149355637, + "grad_norm": 1.9729790687561035, + "kl": 5.7265625, + "learning_rate": 2.42493530526385e-07, + "loss": 0.3744, + "num_tokens": 1212189494.0, + "reward": 1.83740234375, + "reward_std": 0.48765993118286133, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.8671875, + "rewards/format_reward/std": 0.33970388770103455, + "rewards/tag_count_reward/mean": 0.93115234375, + "rewards/tag_count_reward/std": 0.19032683968544006, + "step": 2243 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2019.0, + "completions/mean_length": 823.65625, + "completions/mean_terminated_length": 786.7042236328125, + "completions/min_length": 5.0, + "completions/min_terminated_length": 5.0, + "epoch": 0.7660663992489545, + "grad_norm": 1.0354989767074585, + "kl": 5.359375, + "learning_rate": 2.421023401301636e-07, + "loss": 0.3262, + "num_tokens": 1212696454.0, + "reward": 1.85009765625, + "reward_std": 0.5369447469711304, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.83203125, + "rewards/format_reward/std": 0.374204158782959, + "rewards/tag_count_reward/mean": 0.92041015625, + "rewards/tag_count_reward/std": 0.19817768037319183, + "step": 2244 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2023.0, + "completions/mean_length": 843.009765625, + "completions/mean_terminated_length": 814.0900268554688, + "completions/min_length": 134.0, + "completions/min_terminated_length": 134.0, + "epoch": 0.7664077835623453, + "grad_norm": 1.4246184825897217, + "kl": 5.5546875, + "learning_rate": 2.417115867383553e-07, + "loss": 0.317, + "num_tokens": 1213208827.0, + "reward": 1.82373046875, + "reward_std": 0.4728469252586365, + "rewards/accuracy_reward/mean": 0.03427419438958168, + "rewards/accuracy_reward/std": 0.18211629986763, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.92919921875, + "rewards/tag_count_reward/std": 0.18766237795352936, + "step": 2245 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1871.0, + "completions/mean_length": 812.123046875, + "completions/mean_terminated_length": 764.4928588867188, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "epoch": 0.7667491678757361, + "grad_norm": 2.089632272720337, + "kl": 7.046875, + "learning_rate": 2.4132127090556265e-07, + "loss": 0.4163, + "num_tokens": 1213712970.0, + "reward": 1.8505859375, + "reward_std": 0.6257754564285278, + "rewards/accuracy_reward/mean": 0.11328125, + "rewards/accuracy_reward/std": 0.3172462284564972, + "rewards/format_reward/mean": 0.826171875, + "rewards/format_reward/std": 0.3793322443962097, + "rewards/tag_count_reward/mean": 0.9111328125, + "rewards/tag_count_reward/std": 0.21021628379821777, + "step": 2246 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1998.0, + "completions/mean_length": 791.880859375, + "completions/mean_terminated_length": 748.741455078125, + "completions/min_length": 33.0, + "completions/min_terminated_length": 33.0, + "epoch": 0.7670905521891269, + "grad_norm": 1.114135503768921, + "kl": 6.640625, + "learning_rate": 2.4093139318576793e-07, + "loss": 0.4021, + "num_tokens": 1214196013.0, + "reward": 1.83935546875, + "reward_std": 0.5053403973579407, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.93310546875, + "rewards/tag_count_reward/std": 0.1845095157623291, + "step": 2247 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1933.0, + "completions/mean_length": 795.109375, + "completions/mean_terminated_length": 741.5234375, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "epoch": 0.7674319365025177, + "grad_norm": 1.6775933504104614, + "kl": 7.2265625, + "learning_rate": 2.405419541323314e-07, + "loss": 0.4287, + "num_tokens": 1214684757.0, + "reward": 1.849609375, + "reward_std": 0.5888253450393677, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.916015625, + "rewards/tag_count_reward/std": 0.21164102852344513, + "step": 2248 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2028.0, + "completions/mean_length": 815.65625, + "completions/mean_terminated_length": 781.0120239257812, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "epoch": 0.7677733208159085, + "grad_norm": 2.696317672729492, + "kl": 6.54296875, + "learning_rate": 2.4015295429799e-07, + "loss": 0.3596, + "num_tokens": 1215179813.0, + "reward": 1.8779296875, + "reward_std": 0.5376981496810913, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.9267578125, + "rewards/tag_count_reward/std": 0.1959892213344574, + "step": 2249 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2019.0, + "completions/mean_length": 914.142578125, + "completions/mean_terminated_length": 865.647705078125, + "completions/min_length": 238.0, + "completions/min_terminated_length": 238.0, + "epoch": 0.7681147051292994, + "grad_norm": 1.5230695009231567, + "kl": 7.40625, + "learning_rate": 2.397643942348584e-07, + "loss": 0.4488, + "num_tokens": 1215723390.0, + "reward": 1.79248046875, + "reward_std": 0.6309556365013123, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.818359375, + "rewards/format_reward/std": 0.38592514395713806, + "rewards/tag_count_reward/mean": 0.90771484375, + "rewards/tag_count_reward/std": 0.21193371713161469, + "step": 2250 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1863.0, + "completions/mean_length": 840.552734375, + "completions/mean_terminated_length": 791.469482421875, + "completions/min_length": 152.0, + "completions/min_terminated_length": 152.0, + "epoch": 0.7684560894426901, + "grad_norm": 1.3939034938812256, + "kl": 6.22265625, + "learning_rate": 2.393762744944261e-07, + "loss": 0.3971, + "num_tokens": 1216230905.0, + "reward": 1.8212890625, + "reward_std": 0.523719072341919, + "rewards/accuracy_reward/mean": 0.03125, + "rewards/accuracy_reward/std": 0.17416280508041382, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.9287109375, + "rewards/tag_count_reward/std": 0.1935756355524063, + "step": 2251 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1974.0, + "completions/mean_length": 839.083984375, + "completions/mean_terminated_length": 815.0020141601562, + "completions/min_length": 172.0, + "completions/min_terminated_length": 172.0, + "epoch": 0.7687974737560809, + "grad_norm": 1.0364001989364624, + "kl": 5.78125, + "learning_rate": 2.389885956275585e-07, + "loss": 0.3586, + "num_tokens": 1216740964.0, + "reward": 1.83154296875, + "reward_std": 0.5374894142150879, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.92724609375, + "rewards/tag_count_reward/std": 0.1958593726158142, + "step": 2252 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1950.0, + "completions/mean_length": 793.83984375, + "completions/mean_terminated_length": 773.9325561523438, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "epoch": 0.7691388580694717, + "grad_norm": 1.384509563446045, + "kl": 5.0625, + "learning_rate": 2.386013581844945e-07, + "loss": 0.303, + "num_tokens": 1217236050.0, + "reward": 1.8642578125, + "reward_std": 0.4752752482891083, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.9404296875, + "rewards/tag_count_reward/std": 0.17448893189430237, + "step": 2253 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1961.0, + "completions/mean_length": 816.7265625, + "completions/mean_terminated_length": 779.5653686523438, + "completions/min_length": 123.0, + "completions/min_terminated_length": 123.0, + "epoch": 0.7694802423828625, + "grad_norm": 1.182735800743103, + "kl": 6.3359375, + "learning_rate": 2.3821456271484704e-07, + "loss": 0.383, + "num_tokens": 1217732006.0, + "reward": 1.82470703125, + "reward_std": 0.5288053154945374, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.92431640625, + "rewards/tag_count_reward/std": 0.1953708976507187, + "step": 2254 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2004.0, + "completions/mean_length": 756.423828125, + "completions/mean_terminated_length": 730.6952514648438, + "completions/min_length": 176.0, + "completions/min_terminated_length": 176.0, + "epoch": 0.7698216266962533, + "grad_norm": 1.7976510524749756, + "kl": 4.87109375, + "learning_rate": 2.3782820976760153e-07, + "loss": 0.3435, + "num_tokens": 1218187535.0, + "reward": 1.99560546875, + "reward_std": 0.4458416700363159, + "rewards/accuracy_reward/mean": 0.126953125, + "rewards/accuracy_reward/std": 0.33324605226516724, + "rewards/format_reward/mean": 0.91015625, + "rewards/format_reward/std": 0.2862374484539032, + "rewards/tag_count_reward/mean": 0.95849609375, + "rewards/tag_count_reward/std": 0.1543913036584854, + "step": 2255 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1950.0, + "completions/mean_length": 823.265625, + "completions/mean_terminated_length": 776.0648803710938, + "completions/min_length": 129.0, + "completions/min_terminated_length": 129.0, + "epoch": 0.7701630110096441, + "grad_norm": 0.7203352451324463, + "kl": 5.7578125, + "learning_rate": 2.374422998911151e-07, + "loss": 0.3601, + "num_tokens": 1218689927.0, + "reward": 1.880859375, + "reward_std": 0.4321858286857605, + "rewards/accuracy_reward/mean": 0.04032257944345474, + "rewards/accuracy_reward/std": 0.19691328704357147, + "rewards/format_reward/mean": 0.904296875, + "rewards/format_reward/std": 0.2944713830947876, + "rewards/tag_count_reward/mean": 0.9375, + "rewards/tag_count_reward/std": 0.1843961924314499, + "step": 2256 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 843.36328125, + "completions/mean_terminated_length": 811.97998046875, + "completions/min_length": 124.0, + "completions/min_terminated_length": 124.0, + "epoch": 0.7705043953230349, + "grad_norm": 1.8895782232284546, + "kl": 5.0390625, + "learning_rate": 2.3705683363311656e-07, + "loss": 0.3478, + "num_tokens": 1219197249.0, + "reward": 1.908203125, + "reward_std": 0.4586414396762848, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.896484375, + "rewards/format_reward/std": 0.30492907762527466, + "rewards/tag_count_reward/mean": 0.943359375, + "rewards/tag_count_reward/std": 0.1733599156141281, + "step": 2257 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2011.0, + "completions/mean_length": 793.478515625, + "completions/mean_terminated_length": 745.1298217773438, + "completions/min_length": 144.0, + "completions/min_terminated_length": 144.0, + "epoch": 0.7708457796364258, + "grad_norm": 1.2977958917617798, + "kl": 7.59375, + "learning_rate": 2.3667181154070443e-07, + "loss": 0.4755, + "num_tokens": 1219677638.0, + "reward": 1.81689453125, + "reward_std": 0.4978317320346832, + "rewards/accuracy_reward/mean": 0.032258063554763794, + "rewards/accuracy_reward/std": 0.17686307430267334, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.93017578125, + "rewards/tag_count_reward/std": 0.1847475916147232, + "step": 2258 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1957.0, + "completions/mean_length": 842.4609375, + "completions/mean_terminated_length": 811.0541381835938, + "completions/min_length": 154.0, + "completions/min_terminated_length": 154.0, + "epoch": 0.7711871639498165, + "grad_norm": 1.7056665420532227, + "kl": 7.6796875, + "learning_rate": 2.3628723416034742e-07, + "loss": 0.4458, + "num_tokens": 1220177330.0, + "reward": 1.83935546875, + "reward_std": 0.6094324588775635, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.91943359375, + "rewards/tag_count_reward/std": 0.19778190553188324, + "step": 2259 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2021.0, + "completions/mean_length": 805.095703125, + "completions/mean_terminated_length": 754.5711059570312, + "completions/min_length": 151.0, + "completions/min_terminated_length": 151.0, + "epoch": 0.7715285482632073, + "grad_norm": 2.854001998901367, + "kl": 8.21875, + "learning_rate": 2.359031020378827e-07, + "loss": 0.5032, + "num_tokens": 1220664371.0, + "reward": 1.85986328125, + "reward_std": 0.6096511483192444, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.91259765625, + "rewards/tag_count_reward/std": 0.22019492089748383, + "step": 2260 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1996.0, + "completions/mean_length": 806.982421875, + "completions/mean_terminated_length": 766.9495849609375, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "epoch": 0.7718699325765981, + "grad_norm": 2.7655861377716064, + "kl": 8.59375, + "learning_rate": 2.3551941571851534e-07, + "loss": 0.5227, + "num_tokens": 1221145738.0, + "reward": 1.78662109375, + "reward_std": 0.5410267114639282, + "rewards/accuracy_reward/mean": 0.033203125, + "rewards/accuracy_reward/std": 0.17934183776378632, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.91552734375, + "rewards/tag_count_reward/std": 0.20045962929725647, + "step": 2261 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1934.0, + "completions/mean_length": 767.58984375, + "completions/mean_terminated_length": 728.9456787109375, + "completions/min_length": 145.0, + "completions/min_terminated_length": 145.0, + "epoch": 0.7722113168899889, + "grad_norm": 1.3708407878875732, + "kl": 6.8515625, + "learning_rate": 2.3513617574681828e-07, + "loss": 0.4571, + "num_tokens": 1221610312.0, + "reward": 1.814453125, + "reward_std": 0.558542013168335, + "rewards/accuracy_reward/mean": 0.04838709533214569, + "rewards/accuracy_reward/std": 0.21479946374893188, + "rewards/format_reward/mean": 0.849609375, + "rewards/format_reward/std": 0.35780346393585205, + "rewards/tag_count_reward/mean": 0.91796875, + "rewards/tag_count_reward/std": 0.20538106560707092, + "step": 2262 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1919.0, + "completions/mean_length": 822.24609375, + "completions/mean_terminated_length": 772.4186401367188, + "completions/min_length": 131.0, + "completions/min_terminated_length": 131.0, + "epoch": 0.7725527012033797, + "grad_norm": 1.4335846900939941, + "kl": 6.8203125, + "learning_rate": 2.347533826667302e-07, + "loss": 0.4465, + "num_tokens": 1222102950.0, + "reward": 1.7841796875, + "reward_std": 0.5437231659889221, + "rewards/accuracy_reward/mean": 0.037109375, + "rewards/accuracy_reward/std": 0.18921469151973724, + "rewards/format_reward/mean": 0.8359375, + "rewards/format_reward/std": 0.37069445848464966, + "rewards/tag_count_reward/mean": 0.9111328125, + "rewards/tag_count_reward/std": 0.20610326528549194, + "step": 2263 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1979.0, + "completions/mean_length": 780.509765625, + "completions/mean_terminated_length": 734.325927734375, + "completions/min_length": 110.0, + "completions/min_terminated_length": 110.0, + "epoch": 0.7728940855167705, + "grad_norm": 1.5141960382461548, + "kl": 6.23828125, + "learning_rate": 2.3437103702155617e-07, + "loss": 0.3974, + "num_tokens": 1222576635.0, + "reward": 1.82568359375, + "reward_std": 0.5265644788742065, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.859375, + "rewards/format_reward/std": 0.3479743003845215, + "rewards/tag_count_reward/mean": 0.91943359375, + "rewards/tag_count_reward/std": 0.19901487231254578, + "step": 2264 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2006.0, + "completions/mean_length": 716.271484375, + "completions/mean_terminated_length": 681.5771484375, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "epoch": 0.7732354698301613, + "grad_norm": 1.5562620162963867, + "kl": 5.4453125, + "learning_rate": 2.339891393539656e-07, + "loss": 0.3614, + "num_tokens": 1223013670.0, + "reward": 1.91015625, + "reward_std": 0.5030232667922974, + "rewards/accuracy_reward/mean": 0.123046875, + "rewards/accuracy_reward/std": 0.32881227135658264, + "rewards/format_reward/mean": 0.857421875, + "rewards/format_reward/std": 0.3499840497970581, + "rewards/tag_count_reward/mean": 0.9296875, + "rewards/tag_count_reward/std": 0.18489299714565277, + "step": 2265 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1862.0, + "completions/mean_length": 770.376953125, + "completions/mean_terminated_length": 744.9263305664062, + "completions/min_length": 147.0, + "completions/min_terminated_length": 147.0, + "epoch": 0.7735768541435522, + "grad_norm": 1.3630690574645996, + "kl": 4.7421875, + "learning_rate": 2.336076902059927e-07, + "loss": 0.3269, + "num_tokens": 1223480823.0, + "reward": 1.94140625, + "reward_std": 0.4669956564903259, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.900390625, + "rewards/format_reward/std": 0.29977133870124817, + "rewards/tag_count_reward/mean": 0.951171875, + "rewards/tag_count_reward/std": 0.15736515820026398, + "step": 2266 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1978.0, + "completions/mean_length": 767.064453125, + "completions/mean_terminated_length": 736.322021484375, + "completions/min_length": 164.0, + "completions/min_terminated_length": 164.0, + "epoch": 0.7739182384569429, + "grad_norm": 2.2570767402648926, + "kl": 5.08203125, + "learning_rate": 2.3322669011903461e-07, + "loss": 0.3812, + "num_tokens": 1223951048.0, + "reward": 1.92822265625, + "reward_std": 0.5108921527862549, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.880859375, + "rewards/format_reward/std": 0.32427072525024414, + "rewards/tag_count_reward/mean": 0.94189453125, + "rewards/tag_count_reward/std": 0.16601619124412537, + "step": 2267 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1953.0, + "completions/mean_length": 735.439453125, + "completions/mean_terminated_length": 717.2455444335938, + "completions/min_length": 130.0, + "completions/min_terminated_length": 130.0, + "epoch": 0.7742596227703337, + "grad_norm": 1.4364755153656006, + "kl": 4.8203125, + "learning_rate": 2.3284613963385113e-07, + "loss": 0.3042, + "num_tokens": 1224403225.0, + "reward": 1.921875, + "reward_std": 0.5390156507492065, + "rewards/accuracy_reward/mean": 0.125, + "rewards/accuracy_reward/std": 0.3310423493385315, + "rewards/format_reward/mean": 0.86328125, + "rewards/format_reward/std": 0.3438861668109894, + "rewards/tag_count_reward/mean": 0.93359375, + "rewards/tag_count_reward/std": 0.18369008600711823, + "step": 2268 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1796.0, + "completions/mean_length": 717.837890625, + "completions/mean_terminated_length": 677.692138671875, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "epoch": 0.7746010070837245, + "grad_norm": 1.0798249244689941, + "kl": 5.5625, + "learning_rate": 2.3246603929056435e-07, + "loss": 0.3721, + "num_tokens": 1224847286.0, + "reward": 1.9267578125, + "reward_std": 0.49401456117630005, + "rewards/accuracy_reward/mean": 0.109375, + "rewards/accuracy_reward/std": 0.31241437792778015, + "rewards/format_reward/mean": 0.880859375, + "rewards/format_reward/std": 0.32427072525024414, + "rewards/tag_count_reward/mean": 0.9365234375, + "rewards/tag_count_reward/std": 0.18406164646148682, + "step": 2269 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1985.0, + "completions/mean_length": 749.353515625, + "completions/mean_terminated_length": 712.8453369140625, + "completions/min_length": 128.0, + "completions/min_terminated_length": 128.0, + "epoch": 0.7749423913971153, + "grad_norm": 3.0914320945739746, + "kl": 4.671875, + "learning_rate": 2.320863896286569e-07, + "loss": 0.3542, + "num_tokens": 1225304939.0, + "reward": 1.97705078125, + "reward_std": 0.5090059638023376, + "rewards/accuracy_reward/mean": 0.123046875, + "rewards/accuracy_reward/std": 0.32881227135658264, + "rewards/format_reward/mean": 0.90625, + "rewards/format_reward/std": 0.29176566004753113, + "rewards/tag_count_reward/mean": 0.94775390625, + "rewards/tag_count_reward/std": 0.17155851423740387, + "step": 2270 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1989.0, + "completions/mean_length": 797.6875, + "completions/mean_terminated_length": 770.2355346679688, + "completions/min_length": 84.0, + "completions/min_terminated_length": 84.0, + "epoch": 0.7752837757105061, + "grad_norm": 2.688586711883545, + "kl": 5.3984375, + "learning_rate": 2.3170719118697228e-07, + "loss": 0.3739, + "num_tokens": 1225789195.0, + "reward": 1.8974609375, + "reward_std": 0.4912329316139221, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.8828125, + "rewards/format_reward/std": 0.32195815443992615, + "rewards/tag_count_reward/mean": 0.9384765625, + "rewards/tag_count_reward/std": 0.18272781372070312, + "step": 2271 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1780.0, + "completions/mean_length": 726.333984375, + "completions/mean_terminated_length": 697.3153686523438, + "completions/min_length": 147.0, + "completions/min_terminated_length": 147.0, + "epoch": 0.7756251600238969, + "grad_norm": 1.342585563659668, + "kl": 4.94140625, + "learning_rate": 2.3132844450371314e-07, + "loss": 0.3113, + "num_tokens": 1226235382.0, + "reward": 1.884765625, + "reward_std": 0.4904659390449524, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.87890625, + "rewards/format_reward/std": 0.3265552520751953, + "rewards/tag_count_reward/mean": 0.943359375, + "rewards/tag_count_reward/std": 0.1712302714586258, + "step": 2272 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2021.0, + "completions/mean_length": 825.859375, + "completions/mean_terminated_length": 763.1211547851562, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "epoch": 0.7759665443372877, + "grad_norm": 2.138890027999878, + "kl": 7.7265625, + "learning_rate": 2.3095015011644128e-07, + "loss": 0.4599, + "num_tokens": 1226730526.0, + "reward": 1.822265625, + "reward_std": 0.5565134286880493, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.8359375, + "rewards/format_reward/std": 0.37069445848464966, + "rewards/tag_count_reward/mean": 0.919921875, + "rewards/tag_count_reward/std": 0.20674438774585724, + "step": 2273 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1933.0, + "completions/mean_length": 822.09765625, + "completions/mean_terminated_length": 785.0985717773438, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "epoch": 0.7763079286506785, + "grad_norm": 0.9268271923065186, + "kl": 5.9375, + "learning_rate": 2.3057230856207633e-07, + "loss": 0.3569, + "num_tokens": 1227230800.0, + "reward": 1.85595703125, + "reward_std": 0.5136551856994629, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.93212890625, + "rewards/tag_count_reward/std": 0.18939071893692017, + "step": 2274 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2011.0, + "completions/mean_length": 863.791015625, + "completions/mean_terminated_length": 815.6524047851562, + "completions/min_length": 101.0, + "completions/min_terminated_length": 101.0, + "epoch": 0.7766493129640692, + "grad_norm": 1.4397482872009277, + "kl": 6.6484375, + "learning_rate": 2.3019492037689518e-07, + "loss": 0.3913, + "num_tokens": 1227747429.0, + "reward": 1.82470703125, + "reward_std": 0.5017789602279663, + "rewards/accuracy_reward/mean": 0.029296875, + "rewards/accuracy_reward/std": 0.16880230605602264, + "rewards/format_reward/mean": 0.8671875, + "rewards/format_reward/std": 0.33970388770103455, + "rewards/tag_count_reward/mean": 0.92822265625, + "rewards/tag_count_reward/std": 0.1974627822637558, + "step": 2275 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1911.0, + "completions/mean_length": 770.060546875, + "completions/mean_terminated_length": 726.1717529296875, + "completions/min_length": 96.0, + "completions/min_terminated_length": 96.0, + "epoch": 0.7769906972774601, + "grad_norm": 2.095837116241455, + "kl": 7.59375, + "learning_rate": 2.2981798609653148e-07, + "loss": 0.4539, + "num_tokens": 1228215572.0, + "reward": 1.8271484375, + "reward_std": 0.542022705078125, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.9267578125, + "rewards/tag_count_reward/std": 0.18964596092700958, + "step": 2276 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1966.0, + "completions/mean_length": 818.544921875, + "completions/mean_terminated_length": 786.5150146484375, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "epoch": 0.7773320815908509, + "grad_norm": 0.8329452872276306, + "kl": 6.234375, + "learning_rate": 2.294415062559743e-07, + "loss": 0.3796, + "num_tokens": 1228705499.0, + "reward": 1.9345703125, + "reward_std": 0.5677171349525452, + "rewards/accuracy_reward/mean": 0.1391129046678543, + "rewards/accuracy_reward/std": 0.3464137017726898, + "rewards/format_reward/mean": 0.8671875, + "rewards/format_reward/std": 0.33970388770103455, + "rewards/tag_count_reward/mean": 0.9326171875, + "rewards/tag_count_reward/std": 0.18399934470653534, + "step": 2277 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1992.0, + "completions/mean_length": 820.166015625, + "completions/mean_terminated_length": 777.9979858398438, + "completions/min_length": 41.0, + "completions/min_terminated_length": 41.0, + "epoch": 0.7776734659042417, + "grad_norm": 2.3972365856170654, + "kl": 7.7578125, + "learning_rate": 2.2906548138956815e-07, + "loss": 0.4528, + "num_tokens": 1229203488.0, + "reward": 1.810546875, + "reward_std": 0.5649162530899048, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.919921875, + "rewards/tag_count_reward/std": 0.20733514428138733, + "step": 2278 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2020.0, + "completions/mean_length": 824.634765625, + "completions/mean_terminated_length": 785.1713256835938, + "completions/min_length": 127.0, + "completions/min_terminated_length": 127.0, + "epoch": 0.7780148502176325, + "grad_norm": 1.5216360092163086, + "kl": 6.890625, + "learning_rate": 2.2868991203101145e-07, + "loss": 0.4244, + "num_tokens": 1229703557.0, + "reward": 1.8779296875, + "reward_std": 0.5143810510635376, + "rewards/accuracy_reward/mean": 0.07500000298023224, + "rewards/accuracy_reward/std": 0.26366615295410156, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.9384765625, + "rewards/tag_count_reward/std": 0.17660093307495117, + "step": 2279 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1906.0, + "completions/mean_length": 782.357421875, + "completions/mean_terminated_length": 736.2409057617188, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "epoch": 0.7783562345310233, + "grad_norm": 2.0146689414978027, + "kl": 7.171875, + "learning_rate": 2.283147987133561e-07, + "loss": 0.4427, + "num_tokens": 1230185004.0, + "reward": 1.8876953125, + "reward_std": 0.5053796172142029, + "rewards/accuracy_reward/mean": 0.099609375, + "rewards/accuracy_reward/std": 0.29977133870124817, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.9326171875, + "rewards/tag_count_reward/std": 0.18399934470653534, + "step": 2280 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 847.841796875, + "completions/mean_terminated_length": 788.8175659179688, + "completions/min_length": 128.0, + "completions/min_terminated_length": 128.0, + "epoch": 0.7786976188444141, + "grad_norm": 1.562910795211792, + "kl": 7.0859375, + "learning_rate": 2.2794014196900704e-07, + "loss": 0.4151, + "num_tokens": 1230690843.0, + "reward": 1.85498046875, + "reward_std": 0.5231390595436096, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.91943359375, + "rewards/tag_count_reward/std": 0.20506852865219116, + "step": 2281 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2023.0, + "completions/mean_length": 853.474609375, + "completions/mean_terminated_length": 814.9415283203125, + "completions/min_length": 126.0, + "completions/min_terminated_length": 126.0, + "epoch": 0.7790390031578049, + "grad_norm": 1.12908136844635, + "kl": 5.359375, + "learning_rate": 2.275659423297208e-07, + "loss": 0.3564, + "num_tokens": 1231210094.0, + "reward": 1.9248046875, + "reward_std": 0.4759534001350403, + "rewards/accuracy_reward/mean": 0.103515625, + "rewards/accuracy_reward/std": 0.30492907762527466, + "rewards/format_reward/mean": 0.876953125, + "rewards/format_reward/std": 0.32881227135658264, + "rewards/tag_count_reward/mean": 0.9443359375, + "rewards/tag_count_reward/std": 0.17011895775794983, + "step": 2282 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1982.0, + "completions/mean_length": 826.4921875, + "completions/mean_terminated_length": 794.6693725585938, + "completions/min_length": 160.0, + "completions/min_terminated_length": 160.0, + "epoch": 0.7793803874711956, + "grad_norm": 1.2648156881332397, + "kl": 5.3984375, + "learning_rate": 2.2719220032660553e-07, + "loss": 0.3422, + "num_tokens": 1231711818.0, + "reward": 1.8525390625, + "reward_std": 0.49191102385520935, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.875, + "rewards/format_reward/std": 0.3310423493385315, + "rewards/tag_count_reward/mean": 0.9365234375, + "rewards/tag_count_reward/std": 0.18538589775562286, + "step": 2283 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1940.0, + "completions/mean_length": 771.978515625, + "completions/mean_terminated_length": 728.1555786132812, + "completions/min_length": 47.0, + "completions/min_terminated_length": 47.0, + "epoch": 0.7797217717845865, + "grad_norm": 1.8786842823028564, + "kl": 5.96875, + "learning_rate": 2.2681891649011942e-07, + "loss": 0.3983, + "num_tokens": 1232181631.0, + "reward": 1.87109375, + "reward_std": 0.5296258330345154, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.93359375, + "rewards/tag_count_reward/std": 0.18633443117141724, + "step": 2284 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1772.0, + "completions/mean_length": 828.51171875, + "completions/mean_terminated_length": 794.2288818359375, + "completions/min_length": 165.0, + "completions/min_terminated_length": 165.0, + "epoch": 0.7800631560979773, + "grad_norm": 1.1072237491607666, + "kl": 4.97265625, + "learning_rate": 2.2644609135007088e-07, + "loss": 0.2721, + "num_tokens": 1232684645.0, + "reward": 1.87158203125, + "reward_std": 0.5229383707046509, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.93408203125, + "rewards/tag_count_reward/std": 0.18486134707927704, + "step": 2285 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1884.0, + "completions/mean_length": 882.921875, + "completions/mean_terminated_length": 835.5609741210938, + "completions/min_length": 186.0, + "completions/min_terminated_length": 186.0, + "epoch": 0.7804045404113681, + "grad_norm": 0.6721342206001282, + "kl": 4.5859375, + "learning_rate": 2.2607372543561681e-07, + "loss": 0.2602, + "num_tokens": 1233210765.0, + "reward": 1.8681640625, + "reward_std": 0.44373592734336853, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.873046875, + "rewards/format_reward/std": 0.33324605226516724, + "rewards/tag_count_reward/mean": 0.9345703125, + "rewards/tag_count_reward/std": 0.18863557279109955, + "step": 2286 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1955.0, + "completions/mean_length": 804.482421875, + "completions/mean_terminated_length": 772.086181640625, + "completions/min_length": 158.0, + "completions/min_terminated_length": 158.0, + "epoch": 0.7807459247247589, + "grad_norm": 2.7682130336761475, + "kl": 5.578125, + "learning_rate": 2.257018192752625e-07, + "loss": 0.3786, + "num_tokens": 1233696836.0, + "reward": 1.86962890625, + "reward_std": 0.48753321170806885, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.8828125, + "rewards/format_reward/std": 0.32195815443992615, + "rewards/tag_count_reward/mean": 0.93994140625, + "rewards/tag_count_reward/std": 0.18553723394870758, + "step": 2287 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1892.0, + "completions/mean_length": 881.515625, + "completions/mean_terminated_length": 831.6253051757812, + "completions/min_length": 96.0, + "completions/min_terminated_length": 96.0, + "epoch": 0.7810873090381497, + "grad_norm": 1.2741656303405762, + "kl": 6.0234375, + "learning_rate": 2.2533037339686085e-07, + "loss": 0.3644, + "num_tokens": 1234221676.0, + "reward": 1.81201171875, + "reward_std": 0.5945438146591187, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.818359375, + "rewards/format_reward/std": 0.38592514395713806, + "rewards/tag_count_reward/mean": 0.91748046875, + "rewards/tag_count_reward/std": 0.19697342813014984, + "step": 2288 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2020.0, + "completions/mean_length": 837.576171875, + "completions/mean_terminated_length": 798.5302124023438, + "completions/min_length": 155.0, + "completions/min_terminated_length": 155.0, + "epoch": 0.7814286933515405, + "grad_norm": 1.164945125579834, + "kl": 5.24609375, + "learning_rate": 2.2495938832761114e-07, + "loss": 0.3163, + "num_tokens": 1234734179.0, + "reward": 1.8564453125, + "reward_std": 0.47739219665527344, + "rewards/accuracy_reward/mean": 0.05443548411130905, + "rewards/accuracy_reward/std": 0.227104052901268, + "rewards/format_reward/mean": 0.86328125, + "rewards/format_reward/std": 0.3438861668109894, + "rewards/tag_count_reward/mean": 0.9404296875, + "rewards/tag_count_reward/std": 0.1765792965888977, + "step": 2289 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1971.0, + "completions/mean_length": 856.666015625, + "completions/mean_terminated_length": 808.23779296875, + "completions/min_length": 142.0, + "completions/min_terminated_length": 142.0, + "epoch": 0.7817700776649313, + "grad_norm": 1.6337648630142212, + "kl": 6.9453125, + "learning_rate": 2.245888645940591e-07, + "loss": 0.3918, + "num_tokens": 1235254088.0, + "reward": 1.81005859375, + "reward_std": 0.5663132667541504, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.828125, + "rewards/format_reward/std": 0.3776407241821289, + "rewards/tag_count_reward/mean": 0.91357421875, + "rewards/tag_count_reward/std": 0.20802472531795502, + "step": 2290 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1981.0, + "completions/mean_length": 841.869140625, + "completions/mean_terminated_length": 797.9210815429688, + "completions/min_length": 139.0, + "completions/min_terminated_length": 139.0, + "epoch": 0.782111461978322, + "grad_norm": 1.6514548063278198, + "kl": 7.3046875, + "learning_rate": 2.2421880272209524e-07, + "loss": 0.4617, + "num_tokens": 1235755253.0, + "reward": 1.92236328125, + "reward_std": 0.5728150010108948, + "rewards/accuracy_reward/mean": 0.146484375, + "rewards/accuracy_reward/std": 0.35393697023391724, + "rewards/format_reward/mean": 0.849609375, + "rewards/format_reward/std": 0.35780346393585205, + "rewards/tag_count_reward/mean": 0.92626953125, + "rewards/tag_count_reward/std": 0.19486647844314575, + "step": 2291 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1950.0, + "completions/mean_length": 823.353515625, + "completions/mean_terminated_length": 768.369384765625, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "epoch": 0.7824528462917129, + "grad_norm": 1.2169692516326904, + "kl": 8.0, + "learning_rate": 2.238492032369546e-07, + "loss": 0.4705, + "num_tokens": 1236258218.0, + "reward": 1.7724609375, + "reward_std": 0.6060072183609009, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.8046875, + "rewards/format_reward/std": 0.3968288004398346, + "rewards/tag_count_reward/mean": 0.9072265625, + "rewards/tag_count_reward/std": 0.21200865507125854, + "step": 2292 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2016.0, + "completions/mean_length": 793.775390625, + "completions/mean_terminated_length": 745.4381103515625, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "epoch": 0.7827942306051037, + "grad_norm": 0.8538448214530945, + "kl": 6.5078125, + "learning_rate": 2.2348006666321633e-07, + "loss": 0.4075, + "num_tokens": 1236745143.0, + "reward": 1.8251953125, + "reward_std": 0.520323634147644, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.9267578125, + "rewards/tag_count_reward/std": 0.18835169076919556, + "step": 2293 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 900.056640625, + "completions/mean_terminated_length": 846.0633544921875, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "epoch": 0.7831356149184945, + "grad_norm": 1.750657558441162, + "kl": 7.5, + "learning_rate": 2.2311139352480196e-07, + "loss": 0.4444, + "num_tokens": 1237275332.0, + "reward": 1.8310546875, + "reward_std": 0.5770972371101379, + "rewards/accuracy_reward/mean": 0.10282257944345474, + "rewards/accuracy_reward/std": 0.30403366684913635, + "rewards/format_reward/mean": 0.8203125, + "rewards/format_reward/std": 0.38430243730545044, + "rewards/tag_count_reward/mean": 0.9111328125, + "rewards/tag_count_reward/std": 0.21021628379821777, + "step": 2294 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 924.345703125, + "completions/mean_terminated_length": 861.7918090820312, + "completions/min_length": 41.0, + "completions/min_terminated_length": 41.0, + "epoch": 0.7834769992318853, + "grad_norm": 0.9420157670974731, + "kl": 7.33203125, + "learning_rate": 2.227431843449759e-07, + "loss": 0.4393, + "num_tokens": 1237825605.0, + "reward": 1.78564453125, + "reward_std": 0.5860618352890015, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.82421875, + "rewards/format_reward/std": 0.3810062110424042, + "rewards/tag_count_reward/mean": 0.91064453125, + "rewards/tag_count_reward/std": 0.2085477113723755, + "step": 2295 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1966.0, + "completions/mean_length": 832.921875, + "completions/mean_terminated_length": 788.6477661132812, + "completions/min_length": 151.0, + "completions/min_terminated_length": 151.0, + "epoch": 0.7838183835452761, + "grad_norm": 1.458985447883606, + "kl": 5.6484375, + "learning_rate": 2.2237543964634343e-07, + "loss": 0.3199, + "num_tokens": 1238330461.0, + "reward": 1.85546875, + "reward_std": 0.4921872913837433, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.857421875, + "rewards/format_reward/std": 0.3499840497970581, + "rewards/tag_count_reward/mean": 0.931640625, + "rewards/tag_count_reward/std": 0.186283141374588, + "step": 2296 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1984.0, + "completions/mean_length": 841.6640625, + "completions/mean_terminated_length": 777.1275634765625, + "completions/min_length": 50.0, + "completions/min_terminated_length": 50.0, + "epoch": 0.7841597678586669, + "grad_norm": 1.8807940483093262, + "kl": 6.3671875, + "learning_rate": 2.2200815995085132e-07, + "loss": 0.4228, + "num_tokens": 1238840401.0, + "reward": 1.84130859375, + "reward_std": 0.6311179995536804, + "rewards/accuracy_reward/mean": 0.126953125, + "rewards/accuracy_reward/std": 0.33324605226516724, + "rewards/format_reward/mean": 0.806640625, + "rewards/format_reward/std": 0.39531853795051575, + "rewards/tag_count_reward/mean": 0.90771484375, + "rewards/tag_count_reward/std": 0.21251004934310913, + "step": 2297 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1982.0, + "completions/mean_length": 853.55078125, + "completions/mean_terminated_length": 797.3701171875, + "completions/min_length": 135.0, + "completions/min_terminated_length": 135.0, + "epoch": 0.7845011521720577, + "grad_norm": 2.3324904441833496, + "kl": 5.7265625, + "learning_rate": 2.2164134577978528e-07, + "loss": 0.3718, + "num_tokens": 1239363915.0, + "reward": 1.8291015625, + "reward_std": 0.5720508694648743, + "rewards/accuracy_reward/mean": 0.06854838877916336, + "rewards/accuracy_reward/std": 0.25293970108032227, + "rewards/format_reward/mean": 0.8359375, + "rewards/format_reward/std": 0.37069445848464966, + "rewards/tag_count_reward/mean": 0.9267578125, + "rewards/tag_count_reward/std": 0.18964596092700958, + "step": 2298 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 917.06640625, + "completions/mean_terminated_length": 868.6965942382812, + "completions/min_length": 184.0, + "completions/min_terminated_length": 184.0, + "epoch": 0.7848425364854484, + "grad_norm": 2.211033582687378, + "kl": 5.43359375, + "learning_rate": 2.2127499765377133e-07, + "loss": 0.3338, + "num_tokens": 1239910349.0, + "reward": 1.8466796875, + "reward_std": 0.5793843269348145, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.9169921875, + "rewards/tag_count_reward/std": 0.2049875557422638, + "step": 2299 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1992.0, + "completions/mean_length": 935.25, + "completions/mean_terminated_length": 870.8759765625, + "completions/min_length": 113.0, + "completions/min_terminated_length": 113.0, + "epoch": 0.7851839207988393, + "grad_norm": 1.2065584659576416, + "kl": 5.4296875, + "learning_rate": 2.2090911609277375e-07, + "loss": 0.3338, + "num_tokens": 1240469373.0, + "reward": 1.80859375, + "reward_std": 0.5164964199066162, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.841796875, + "rewards/format_reward/std": 0.36528825759887695, + "rewards/tag_count_reward/mean": 0.919921875, + "rewards/tag_count_reward/std": 0.20555779337882996, + "step": 2300 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1999.0, + "completions/mean_length": 872.611328125, + "completions/mean_terminated_length": 819.8387451171875, + "completions/min_length": 157.0, + "completions/min_terminated_length": 157.0, + "epoch": 0.7855253051122301, + "grad_norm": 2.977849006652832, + "kl": 6.109375, + "learning_rate": 2.2054370161609415e-07, + "loss": 0.4412, + "num_tokens": 1240997430.0, + "reward": 1.853515625, + "reward_std": 0.5015002489089966, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.931640625, + "rewards/tag_count_reward/std": 0.18889120221138, + "step": 2301 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.060546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1993.0, + "completions/mean_length": 923.1171875, + "completions/mean_terminated_length": 850.6195678710938, + "completions/min_length": 171.0, + "completions/min_terminated_length": 171.0, + "epoch": 0.7858666894256209, + "grad_norm": 2.0174193382263184, + "kl": 7.0078125, + "learning_rate": 2.201787547423719e-07, + "loss": 0.461, + "num_tokens": 1241545890.0, + "reward": 1.8154296875, + "reward_std": 0.5970208644866943, + "rewards/accuracy_reward/mean": 0.099609375, + "rewards/accuracy_reward/std": 0.29977133870124817, + "rewards/format_reward/mean": 0.8046875, + "rewards/format_reward/std": 0.3968288004398346, + "rewards/tag_count_reward/mean": 0.9111328125, + "rewards/tag_count_reward/std": 0.2072867453098297, + "step": 2302 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1972.0, + "completions/mean_length": 898.25390625, + "completions/mean_terminated_length": 858.7677001953125, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "epoch": 0.7862080737390117, + "grad_norm": 0.9348337054252625, + "kl": 5.7734375, + "learning_rate": 2.19814275989582e-07, + "loss": 0.3434, + "num_tokens": 1242082756.0, + "reward": 1.845703125, + "reward_std": 0.5673692226409912, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.921875, + "rewards/tag_count_reward/std": 0.20332364737987518, + "step": 2303 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2013.0, + "completions/mean_length": 868.435546875, + "completions/mean_terminated_length": 825.4555053710938, + "completions/min_length": 114.0, + "completions/min_terminated_length": 114.0, + "epoch": 0.7865494580524025, + "grad_norm": 1.2880990505218506, + "kl": 6.515625, + "learning_rate": 2.1945026587503578e-07, + "loss": 0.3942, + "num_tokens": 1242602003.0, + "reward": 1.8369140625, + "reward_std": 0.5780290365219116, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.9208984375, + "rewards/tag_count_reward/std": 0.20294499397277832, + "step": 2304 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 768.921875, + "completions/mean_terminated_length": 727.6612548828125, + "completions/min_length": 114.0, + "completions/min_terminated_length": 114.0, + "epoch": 0.7868908423657933, + "grad_norm": 1.5181031227111816, + "kl": 7.3359375, + "learning_rate": 2.1908672491537854e-07, + "loss": 0.4568, + "num_tokens": 1243069867.0, + "reward": 1.87353515625, + "reward_std": 0.5800575613975525, + "rewards/accuracy_reward/mean": 0.1088709682226181, + "rewards/accuracy_reward/std": 0.31179171800613403, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.92822265625, + "rewards/tag_count_reward/std": 0.19180715084075928, + "step": 2305 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 797.25390625, + "completions/mean_terminated_length": 772.3386840820312, + "completions/min_length": 15.0, + "completions/min_terminated_length": 15.0, + "epoch": 0.7872322266791841, + "grad_norm": 1.515105128288269, + "kl": 6.1640625, + "learning_rate": 2.187236536265904e-07, + "loss": 0.3612, + "num_tokens": 1243555197.0, + "reward": 1.7978515625, + "reward_std": 0.5367641448974609, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.83203125, + "rewards/format_reward/std": 0.374204158782959, + "rewards/tag_count_reward/mean": 0.9189453125, + "rewards/tag_count_reward/std": 0.1922481507062912, + "step": 2306 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 821.015625, + "completions/mean_terminated_length": 763.3046875, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "epoch": 0.787573610992575, + "grad_norm": 1.7140458822250366, + "kl": 7.6171875, + "learning_rate": 2.1836105252398483e-07, + "loss": 0.4465, + "num_tokens": 1244053477.0, + "reward": 1.83837890625, + "reward_std": 0.6304191946983337, + "rewards/accuracy_reward/mean": 0.115234375, + "rewards/accuracy_reward/std": 0.3196168541908264, + "rewards/format_reward/mean": 0.814453125, + "rewards/format_reward/std": 0.38912075757980347, + "rewards/tag_count_reward/mean": 0.90869140625, + "rewards/tag_count_reward/std": 0.2117803692817688, + "step": 2307 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1978.0, + "completions/mean_length": 844.384765625, + "completions/mean_terminated_length": 820.4083862304688, + "completions/min_length": 29.0, + "completions/min_terminated_length": 29.0, + "epoch": 0.7879149953059656, + "grad_norm": 1.7330269813537598, + "kl": 4.1875, + "learning_rate": 2.1799892212220745e-07, + "loss": 0.2406, + "num_tokens": 1244564410.0, + "reward": 1.92919921875, + "reward_std": 0.5259957313537598, + "rewards/accuracy_reward/mean": 0.109375, + "rewards/accuracy_reward/std": 0.31241437792778015, + "rewards/format_reward/mean": 0.876953125, + "rewards/format_reward/std": 0.32881227135658264, + "rewards/tag_count_reward/mean": 0.94287109375, + "rewards/tag_count_reward/std": 0.16708904504776, + "step": 2308 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1934.0, + "completions/mean_length": 832.71875, + "completions/mean_terminated_length": 778.1550903320312, + "completions/min_length": 5.0, + "completions/min_terminated_length": 5.0, + "epoch": 0.7882563796193565, + "grad_norm": 2.6183900833129883, + "kl": 6.5234375, + "learning_rate": 2.1763726293523642e-07, + "loss": 0.437, + "num_tokens": 1245066602.0, + "reward": 1.8818359375, + "reward_std": 0.5811901092529297, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.9248046875, + "rewards/tag_count_reward/std": 0.19773660600185394, + "step": 2309 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1974.0, + "completions/mean_length": 810.8046875, + "completions/mean_terminated_length": 778.5731811523438, + "completions/min_length": 174.0, + "completions/min_terminated_length": 174.0, + "epoch": 0.7885977639327473, + "grad_norm": 2.6835498809814453, + "kl": 5.421875, + "learning_rate": 2.1727607547638073e-07, + "loss": 0.324, + "num_tokens": 1245556326.0, + "reward": 1.89013671875, + "reward_std": 0.5729560852050781, + "rewards/accuracy_reward/mean": 0.12109375, + "rewards/accuracy_reward/std": 0.3265552520751953, + "rewards/format_reward/mean": 0.841796875, + "rewards/format_reward/std": 0.36528825759887695, + "rewards/tag_count_reward/mean": 0.92724609375, + "rewards/tag_count_reward/std": 0.18625685572624207, + "step": 2310 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1976.0, + "completions/mean_length": 811.447265625, + "completions/mean_terminated_length": 768.9798583984375, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "epoch": 0.7889391482461381, + "grad_norm": 1.9135109186172485, + "kl": 5.8515625, + "learning_rate": 2.1691536025827982e-07, + "loss": 0.3853, + "num_tokens": 1246048139.0, + "reward": 1.90234375, + "reward_std": 0.5492792129516602, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.8671875, + "rewards/format_reward/std": 0.33970388770103455, + "rewards/tag_count_reward/mean": 0.9296875, + "rewards/tag_count_reward/std": 0.1774672269821167, + "step": 2311 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1816.0, + "completions/mean_length": 805.12890625, + "completions/mean_terminated_length": 754.6056518554688, + "completions/min_length": 50.0, + "completions/min_terminated_length": 50.0, + "epoch": 0.7892805325595289, + "grad_norm": 2.771057367324829, + "kl": 4.82421875, + "learning_rate": 2.1655511779290285e-07, + "loss": 0.3316, + "num_tokens": 1246543021.0, + "reward": 1.88232421875, + "reward_std": 0.44993066787719727, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.890625, + "rewards/format_reward/std": 0.31241437792778015, + "rewards/tag_count_reward/mean": 0.94677734375, + "rewards/tag_count_reward/std": 0.16170679032802582, + "step": 2312 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1768.0, + "completions/mean_length": 797.34765625, + "completions/mean_terminated_length": 754.39599609375, + "completions/min_length": 55.0, + "completions/min_terminated_length": 55.0, + "epoch": 0.7896219168729197, + "grad_norm": 2.1502861976623535, + "kl": 5.30078125, + "learning_rate": 2.161953485915483e-07, + "loss": 0.3465, + "num_tokens": 1247032655.0, + "reward": 1.873046875, + "reward_std": 0.4173519015312195, + "rewards/accuracy_reward/mean": 0.029296875, + "rewards/accuracy_reward/std": 0.16880230605602264, + "rewards/format_reward/mean": 0.89453125, + "rewards/format_reward/std": 0.3074568510055542, + "rewards/tag_count_reward/mean": 0.94921875, + "rewards/tag_count_reward/std": 0.1605987250804901, + "step": 2313 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2013.0, + "completions/mean_length": 782.4375, + "completions/mean_terminated_length": 754.6506958007812, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "epoch": 0.7899633011863105, + "grad_norm": 1.537009596824646, + "kl": 5.828125, + "learning_rate": 2.1583605316484286e-07, + "loss": 0.3451, + "num_tokens": 1247508991.0, + "reward": 1.8583984375, + "reward_std": 0.5327895283699036, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.9306640625, + "rewards/tag_count_reward/std": 0.1799030750989914, + "step": 2314 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1718.0, + "completions/mean_length": 759.458984375, + "completions/mean_terminated_length": 723.2349243164062, + "completions/min_length": 121.0, + "completions/min_terminated_length": 121.0, + "epoch": 0.7903046854997013, + "grad_norm": 2.1879611015319824, + "kl": 6.203125, + "learning_rate": 2.1547723202274039e-07, + "loss": 0.3981, + "num_tokens": 1247968394.0, + "reward": 1.83056640625, + "reward_std": 0.5375639200210571, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.92431640625, + "rewards/tag_count_reward/std": 0.18835864961147308, + "step": 2315 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1971.0, + "completions/mean_length": 804.03515625, + "completions/mean_terminated_length": 758.70849609375, + "completions/min_length": 70.0, + "completions/min_terminated_length": 70.0, + "epoch": 0.790646069813092, + "grad_norm": 10.085799217224121, + "kl": 9.140625, + "learning_rate": 2.1511888567452224e-07, + "loss": 0.5064, + "num_tokens": 1248457676.0, + "reward": 1.7333984375, + "reward_std": 0.6078810691833496, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.802734375, + "rewards/format_reward/std": 0.3983237147331238, + "rewards/tag_count_reward/mean": 0.8916015625, + "rewards/tag_count_reward/std": 0.2308577001094818, + "step": 2316 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1905.0, + "completions/mean_length": 755.626953125, + "completions/mean_terminated_length": 711.242431640625, + "completions/min_length": 135.0, + "completions/min_terminated_length": 135.0, + "epoch": 0.7909874541264829, + "grad_norm": 3.328634262084961, + "kl": 6.3984375, + "learning_rate": 2.1476101462879504e-07, + "loss": 0.3507, + "num_tokens": 1248919309.0, + "reward": 1.8779296875, + "reward_std": 0.5386531949043274, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.9287109375, + "rewards/tag_count_reward/std": 0.18910102546215057, + "step": 2317 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2031.0, + "completions/mean_length": 723.43359375, + "completions/mean_terminated_length": 688.9258422851562, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "epoch": 0.7913288384398737, + "grad_norm": 2.646514892578125, + "kl": 6.671875, + "learning_rate": 2.1440361939349166e-07, + "loss": 0.396, + "num_tokens": 1249369083.0, + "reward": 1.87353515625, + "reward_std": 0.5484198927879333, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.92431640625, + "rewards/tag_count_reward/std": 0.18770819902420044, + "step": 2318 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2001.0, + "completions/mean_length": 817.1015625, + "completions/mean_terminated_length": 777.3951416015625, + "completions/min_length": 82.0, + "completions/min_terminated_length": 82.0, + "epoch": 0.7916702227532645, + "grad_norm": 2.307469129562378, + "kl": 6.171875, + "learning_rate": 2.1404670047586905e-07, + "loss": 0.3821, + "num_tokens": 1249868527.0, + "reward": 1.857421875, + "reward_std": 0.4760078191757202, + "rewards/accuracy_reward/mean": 0.04838709533214569, + "rewards/accuracy_reward/std": 0.21479946374893188, + "rewards/format_reward/mean": 0.875, + "rewards/format_reward/std": 0.3310423493385315, + "rewards/tag_count_reward/mean": 0.935546875, + "rewards/tag_count_reward/std": 0.18832378089427948, + "step": 2319 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1766.0, + "completions/mean_length": 766.43359375, + "completions/mean_terminated_length": 733.0460815429688, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "epoch": 0.7920116070666553, + "grad_norm": 2.878211498260498, + "kl": 5.5390625, + "learning_rate": 2.1369025838250847e-07, + "loss": 0.359, + "num_tokens": 1250326109.0, + "reward": 1.9375, + "reward_std": 0.497491717338562, + "rewards/accuracy_reward/mean": 0.1015625, + "rewards/accuracy_reward/std": 0.30236753821372986, + "rewards/format_reward/mean": 0.890625, + "rewards/format_reward/std": 0.31241437792778015, + "rewards/tag_count_reward/mean": 0.9453125, + "rewards/tag_count_reward/std": 0.16899466514587402, + "step": 2320 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1951.0, + "completions/mean_length": 790.7578125, + "completions/mean_terminated_length": 763.1536865234375, + "completions/min_length": 143.0, + "completions/min_terminated_length": 143.0, + "epoch": 0.7923529913800461, + "grad_norm": 0.9542919993400574, + "kl": 4.8671875, + "learning_rate": 2.1333429361931412e-07, + "loss": 0.3083, + "num_tokens": 1250808753.0, + "reward": 1.93359375, + "reward_std": 0.4397716522216797, + "rewards/accuracy_reward/mean": 0.1015625, + "rewards/accuracy_reward/std": 0.30236753821372986, + "rewards/format_reward/mean": 0.884765625, + "rewards/format_reward/std": 0.3196168541908264, + "rewards/tag_count_reward/mean": 0.947265625, + "rewards/tag_count_reward/std": 0.16743822395801544, + "step": 2321 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1971.0, + "completions/mean_length": 752.78515625, + "completions/mean_terminated_length": 732.2262573242188, + "completions/min_length": 114.0, + "completions/min_terminated_length": 114.0, + "epoch": 0.7926943756934369, + "grad_norm": 1.120664358139038, + "kl": 4.78125, + "learning_rate": 2.1297880669151315e-07, + "loss": 0.2742, + "num_tokens": 1251270467.0, + "reward": 1.859375, + "reward_std": 0.41783279180526733, + "rewards/accuracy_reward/mean": 0.025390625, + "rewards/accuracy_reward/std": 0.15746226906776428, + "rewards/format_reward/mean": 0.884765625, + "rewards/format_reward/std": 0.3196168541908264, + "rewards/tag_count_reward/mean": 0.94921875, + "rewards/tag_count_reward/std": 0.15829749405384064, + "step": 2322 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 746.47265625, + "completions/mean_terminated_length": 707.1911010742188, + "completions/min_length": 150.0, + "completions/min_terminated_length": 150.0, + "epoch": 0.7930357600068277, + "grad_norm": 1.0932191610336304, + "kl": 4.9765625, + "learning_rate": 2.1262379810365404e-07, + "loss": 0.2948, + "num_tokens": 1251723477.0, + "reward": 1.9033203125, + "reward_std": 0.45974022150039673, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.88671875, + "rewards/format_reward/std": 0.3172462284564972, + "rewards/tag_count_reward/mean": 0.9404296875, + "rewards/tag_count_reward/std": 0.17094823718070984, + "step": 2323 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 775.20703125, + "completions/mean_terminated_length": 736.792724609375, + "completions/min_length": 82.0, + "completions/min_terminated_length": 82.0, + "epoch": 0.7933771443202184, + "grad_norm": 1.393788456916809, + "kl": 5.609375, + "learning_rate": 2.122692683596064e-07, + "loss": 0.3482, + "num_tokens": 1252198703.0, + "reward": 1.90283203125, + "reward_std": 0.5502181053161621, + "rewards/accuracy_reward/mean": 0.1015625, + "rewards/accuracy_reward/std": 0.30236753821372986, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.93017578125, + "rewards/tag_count_reward/std": 0.19379454851150513, + "step": 2324 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 762.34765625, + "completions/mean_terminated_length": 723.5452270507812, + "completions/min_length": 98.0, + "completions/min_terminated_length": 98.0, + "epoch": 0.7937185286336093, + "grad_norm": 2.35156512260437, + "kl": 5.890625, + "learning_rate": 2.1191521796256067e-07, + "loss": 0.4083, + "num_tokens": 1252664241.0, + "reward": 1.86328125, + "reward_std": 0.49813130497932434, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.8828125, + "rewards/format_reward/std": 0.32195815443992615, + "rewards/tag_count_reward/mean": 0.939453125, + "rewards/tag_count_reward/std": 0.18036192655563354, + "step": 2325 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.005859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1939.0, + "completions/mean_length": 807.81640625, + "completions/mean_terminated_length": 800.5068969726562, + "completions/min_length": 34.0, + "completions/min_terminated_length": 34.0, + "epoch": 0.7940599129470001, + "grad_norm": 1.3092200756072998, + "kl": 4.61328125, + "learning_rate": 2.1156164741502639e-07, + "loss": 0.2665, + "num_tokens": 1253155187.0, + "reward": 1.96533203125, + "reward_std": 0.4647040367126465, + "rewards/accuracy_reward/mean": 0.109375, + "rewards/accuracy_reward/std": 0.31241437792778015, + "rewards/format_reward/mean": 0.908203125, + "rewards/format_reward/std": 0.289021372795105, + "rewards/tag_count_reward/mean": 0.94775390625, + "rewards/tag_count_reward/std": 0.16352833807468414, + "step": 2326 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 744.103515625, + "completions/mean_terminated_length": 726.0297241210938, + "completions/min_length": 72.0, + "completions/min_terminated_length": 72.0, + "epoch": 0.7944012972603909, + "grad_norm": 1.8248697519302368, + "kl": 4.44140625, + "learning_rate": 2.1120855721883253e-07, + "loss": 0.2926, + "num_tokens": 1253610152.0, + "reward": 1.97021484375, + "reward_std": 0.4814947843551636, + "rewards/accuracy_reward/mean": 0.115234375, + "rewards/accuracy_reward/std": 0.3196168541908264, + "rewards/format_reward/mean": 0.90234375, + "rewards/format_reward/std": 0.29713961482048035, + "rewards/tag_count_reward/mean": 0.95263671875, + "rewards/tag_count_reward/std": 0.1566462367773056, + "step": 2327 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1892.0, + "completions/mean_length": 794.828125, + "completions/mean_terminated_length": 769.8645629882812, + "completions/min_length": 75.0, + "completions/min_terminated_length": 75.0, + "epoch": 0.7947426815737817, + "grad_norm": 1.264649748802185, + "kl": 4.9140625, + "learning_rate": 2.1085594787512579e-07, + "loss": 0.2962, + "num_tokens": 1254103168.0, + "reward": 1.90380859375, + "reward_std": 0.4397355318069458, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.8984375, + "rewards/format_reward/std": 0.30236753821372986, + "rewards/tag_count_reward/mean": 0.95068359375, + "rewards/tag_count_reward/std": 0.1666538417339325, + "step": 2328 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2011.0, + "completions/mean_length": 813.044921875, + "completions/mean_terminated_length": 783.4060668945312, + "completions/min_length": 148.0, + "completions/min_terminated_length": 148.0, + "epoch": 0.7950840658871725, + "grad_norm": 1.5095208883285522, + "kl": 5.671875, + "learning_rate": 2.105038198843707e-07, + "loss": 0.3433, + "num_tokens": 1254601383.0, + "reward": 1.93017578125, + "reward_std": 0.5012025833129883, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.890625, + "rewards/format_reward/std": 0.31241437792778015, + "rewards/tag_count_reward/mean": 0.94775390625, + "rewards/tag_count_reward/std": 0.16868269443511963, + "step": 2329 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1962.0, + "completions/mean_length": 786.5078125, + "completions/mean_terminated_length": 751.0441284179688, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "epoch": 0.7954254502005633, + "grad_norm": 1.4137483835220337, + "kl": 6.1875, + "learning_rate": 2.101521737463487e-07, + "loss": 0.3721, + "num_tokens": 1255070907.0, + "reward": 1.955078125, + "reward_std": 0.5198351740837097, + "rewards/accuracy_reward/mean": 0.12890625, + "rewards/accuracy_reward/std": 0.33542385697364807, + "rewards/format_reward/mean": 0.890625, + "rewards/format_reward/std": 0.31241437792778015, + "rewards/tag_count_reward/mean": 0.935546875, + "rewards/tag_count_reward/std": 0.18570774793624878, + "step": 2330 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1841.0, + "completions/mean_length": 796.857421875, + "completions/mean_terminated_length": 769.38720703125, + "completions/min_length": 33.0, + "completions/min_terminated_length": 33.0, + "epoch": 0.7957668345139541, + "grad_norm": 1.2894519567489624, + "kl": 5.2109375, + "learning_rate": 2.0980100996015694e-07, + "loss": 0.3191, + "num_tokens": 1255558130.0, + "reward": 1.88232421875, + "reward_std": 0.4153871536254883, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.8984375, + "rewards/format_reward/std": 0.30236753821372986, + "rewards/tag_count_reward/mean": 0.94482421875, + "rewards/tag_count_reward/std": 0.1762780249118805, + "step": 2331 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1993.0, + "completions/mean_length": 832.83203125, + "completions/mean_terminated_length": 783.4349365234375, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "epoch": 0.7961082188273448, + "grad_norm": 1.486045241355896, + "kl": 5.765625, + "learning_rate": 2.094503290242084e-07, + "loss": 0.345, + "num_tokens": 1256073788.0, + "reward": 1.859375, + "reward_std": 0.4540967345237732, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.876953125, + "rewards/format_reward/std": 0.32881227135658264, + "rewards/tag_count_reward/mean": 0.9375, + "rewards/tag_count_reward/std": 0.18104934692382812, + "step": 2332 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.009765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2030.0, + "completions/mean_length": 789.7265625, + "completions/mean_terminated_length": 777.3175659179688, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "epoch": 0.7964496031407357, + "grad_norm": 0.76603102684021, + "kl": 4.84765625, + "learning_rate": 2.091001314362303e-07, + "loss": 0.2737, + "num_tokens": 1256561504.0, + "reward": 1.93310546875, + "reward_std": 0.4562763571739197, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.90234375, + "rewards/format_reward/std": 0.29713961482048035, + "rewards/tag_count_reward/mean": 0.95263671875, + "rewards/tag_count_reward/std": 0.15428605675697327, + "step": 2333 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1980.0, + "completions/mean_length": 842.16796875, + "completions/mean_terminated_length": 800.755615234375, + "completions/min_length": 139.0, + "completions/min_terminated_length": 139.0, + "epoch": 0.7967909874541265, + "grad_norm": 1.4693493843078613, + "kl": 7.1640625, + "learning_rate": 2.087504176932643e-07, + "loss": 0.4425, + "num_tokens": 1257068758.0, + "reward": 1.83984375, + "reward_std": 0.4927568733692169, + "rewards/accuracy_reward/mean": 0.029296875, + "rewards/accuracy_reward/std": 0.16880230605602264, + "rewards/format_reward/mean": 0.876953125, + "rewards/format_reward/std": 0.32881227135658264, + "rewards/tag_count_reward/mean": 0.93359375, + "rewards/tag_count_reward/std": 0.18829332292079926, + "step": 2334 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2000.0, + "completions/mean_length": 839.037109375, + "completions/mean_terminated_length": 771.7340698242188, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "epoch": 0.7971323717675173, + "grad_norm": 2.0895612239837646, + "kl": 8.5703125, + "learning_rate": 2.0840118829166498e-07, + "loss": 0.5156, + "num_tokens": 1257570265.0, + "reward": 1.84326171875, + "reward_std": 0.5723297595977783, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.91943359375, + "rewards/tag_count_reward/std": 0.20506852865219116, + "step": 2335 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2005.0, + "completions/mean_length": 842.220703125, + "completions/mean_terminated_length": 785.5071411132812, + "completions/min_length": 143.0, + "completions/min_terminated_length": 143.0, + "epoch": 0.7974737560809081, + "grad_norm": 1.2615413665771484, + "kl": 7.96875, + "learning_rate": 2.0805244372709952e-07, + "loss": 0.4921, + "num_tokens": 1258081034.0, + "reward": 1.837890625, + "reward_std": 0.5462841987609863, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.92578125, + "rewards/tag_count_reward/std": 0.19686728715896606, + "step": 2336 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 845.64453125, + "completions/mean_terminated_length": 804.3515625, + "completions/min_length": 125.0, + "completions/min_terminated_length": 125.0, + "epoch": 0.7978151403942989, + "grad_norm": 0.9753010869026184, + "kl": 7.1875, + "learning_rate": 2.077041844945472e-07, + "loss": 0.4314, + "num_tokens": 1258589540.0, + "reward": 1.89111328125, + "reward_std": 0.5716447830200195, + "rewards/accuracy_reward/mean": 0.1015625, + "rewards/accuracy_reward/std": 0.30236753821372986, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.92822265625, + "rewards/tag_count_reward/std": 0.1974627822637558, + "step": 2337 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2016.0, + "completions/mean_length": 825.892578125, + "completions/mean_terminated_length": 765.7888793945312, + "completions/min_length": 131.0, + "completions/min_terminated_length": 131.0, + "epoch": 0.7981565247076897, + "grad_norm": 1.5391372442245483, + "kl": 7.796875, + "learning_rate": 2.0735641108829813e-07, + "loss": 0.5046, + "num_tokens": 1259089293.0, + "reward": 1.82763671875, + "reward_std": 0.5773891806602478, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.849609375, + "rewards/format_reward/std": 0.35780346393585205, + "rewards/tag_count_reward/mean": 0.91748046875, + "rewards/tag_count_reward/std": 0.21019071340560913, + "step": 2338 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2023.0, + "completions/mean_length": 814.05078125, + "completions/mean_terminated_length": 776.808837890625, + "completions/min_length": 89.0, + "completions/min_terminated_length": 89.0, + "epoch": 0.7984979090210805, + "grad_norm": 0.7947782278060913, + "kl": 7.4921875, + "learning_rate": 2.070091240019533e-07, + "loss": 0.4681, + "num_tokens": 1259586087.0, + "reward": 1.810546875, + "reward_std": 0.5679647326469421, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.841796875, + "rewards/format_reward/std": 0.36528825759887695, + "rewards/tag_count_reward/mean": 0.919921875, + "rewards/tag_count_reward/std": 0.20555779337882996, + "step": 2339 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2020.0, + "completions/mean_length": 868.98046875, + "completions/mean_terminated_length": 813.5255737304688, + "completions/min_length": 168.0, + "completions/min_terminated_length": 168.0, + "epoch": 0.7988392933344712, + "grad_norm": 1.0265549421310425, + "kl": 6.1171875, + "learning_rate": 2.066623237284229e-07, + "loss": 0.3741, + "num_tokens": 1260107053.0, + "reward": 1.9287109375, + "reward_std": 0.5877367854118347, + "rewards/accuracy_reward/mean": 0.14453125, + "rewards/accuracy_reward/std": 0.35197147727012634, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.9228515625, + "rewards/tag_count_reward/std": 0.205490380525589, + "step": 2340 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.056640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 904.48046875, + "completions/mean_terminated_length": 835.8219604492188, + "completions/min_length": 156.0, + "completions/min_terminated_length": 156.0, + "epoch": 0.799180677647862, + "grad_norm": 1.025760293006897, + "kl": 7.3671875, + "learning_rate": 2.0631601075992677e-07, + "loss": 0.4521, + "num_tokens": 1260640691.0, + "reward": 1.78173828125, + "reward_std": 0.5654925107955933, + "rewards/accuracy_reward/mean": 0.038306452333927155, + "rewards/accuracy_reward/std": 0.19212882220745087, + "rewards/format_reward/mean": 0.833984375, + "rewards/format_reward/std": 0.3724585771560669, + "rewards/tag_count_reward/mean": 0.91064453125, + "rewards/tag_count_reward/std": 0.21603746712207794, + "step": 2341 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1984.0, + "completions/mean_length": 867.6953125, + "completions/mean_terminated_length": 824.6882934570312, + "completions/min_length": 89.0, + "completions/min_terminated_length": 89.0, + "epoch": 0.7995220619612529, + "grad_norm": 2.2666099071502686, + "kl": 6.3203125, + "learning_rate": 2.0597018558799272e-07, + "loss": 0.3268, + "num_tokens": 1261165735.0, + "reward": 1.83740234375, + "reward_std": 0.5497316718101501, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.91748046875, + "rewards/tag_count_reward/std": 0.20248481631278992, + "step": 2342 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1902.0, + "completions/mean_length": 899.16796875, + "completions/mean_terminated_length": 857.3077392578125, + "completions/min_length": 176.0, + "completions/min_terminated_length": 176.0, + "epoch": 0.7998634462746437, + "grad_norm": 2.0839056968688965, + "kl": 4.3828125, + "learning_rate": 2.056248487034562e-07, + "loss": 0.297, + "num_tokens": 1261706445.0, + "reward": 1.892578125, + "reward_std": 0.4774249196052551, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.890625, + "rewards/format_reward/std": 0.31241437792778015, + "rewards/tag_count_reward/mean": 0.943359375, + "rewards/tag_count_reward/std": 0.1726529598236084, + "step": 2343 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 837.255859375, + "completions/mean_terminated_length": 793.1397094726562, + "completions/min_length": 166.0, + "completions/min_terminated_length": 166.0, + "epoch": 0.8002048305880345, + "grad_norm": 2.329362630844116, + "kl": 5.640625, + "learning_rate": 2.0528000059645995e-07, + "loss": 0.3844, + "num_tokens": 1262214304.0, + "reward": 1.83642578125, + "reward_std": 0.47069251537323, + "rewards/accuracy_reward/mean": 0.033203125, + "rewards/accuracy_reward/std": 0.17934183776378632, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.93212890625, + "rewards/tag_count_reward/std": 0.18874379992485046, + "step": 2344 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1960.0, + "completions/mean_length": 854.923828125, + "completions/mean_terminated_length": 828.728515625, + "completions/min_length": 176.0, + "completions/min_terminated_length": 176.0, + "epoch": 0.8005462149014253, + "grad_norm": 1.5461212396621704, + "kl": 4.9296875, + "learning_rate": 2.0493564175645256e-07, + "loss": 0.3225, + "num_tokens": 1262734457.0, + "reward": 1.92041015625, + "reward_std": 0.5226565599441528, + "rewards/accuracy_reward/mean": 0.107421875, + "rewards/accuracy_reward/std": 0.30995169281959534, + "rewards/format_reward/mean": 0.876953125, + "rewards/format_reward/std": 0.32881227135658264, + "rewards/tag_count_reward/mean": 0.93603515625, + "rewards/tag_count_reward/std": 0.1828918159008026, + "step": 2345 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2018.0, + "completions/mean_length": 901.26171875, + "completions/mean_terminated_length": 842.394287109375, + "completions/min_length": 206.0, + "completions/min_terminated_length": 206.0, + "epoch": 0.8008875992148161, + "grad_norm": 3.50020170211792, + "kl": 5.2265625, + "learning_rate": 2.0459177267218878e-07, + "loss": 0.3717, + "num_tokens": 1263283727.0, + "reward": 1.85595703125, + "reward_std": 0.47672832012176514, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.859375, + "rewards/format_reward/std": 0.3479743003845215, + "rewards/tag_count_reward/mean": 0.93408203125, + "rewards/tag_count_reward/std": 0.18353331089019775, + "step": 2346 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1984.0, + "completions/mean_length": 834.431640625, + "completions/mean_terminated_length": 797.8048095703125, + "completions/min_length": 96.0, + "completions/min_terminated_length": 96.0, + "epoch": 0.8012289835282069, + "grad_norm": 1.2660900354385376, + "kl": 5.4765625, + "learning_rate": 2.042483938317276e-07, + "loss": 0.3455, + "num_tokens": 1263788364.0, + "reward": 1.88525390625, + "reward_std": 0.5068372488021851, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.873046875, + "rewards/format_reward/std": 0.33324605226516724, + "rewards/tag_count_reward/mean": 0.93603515625, + "rewards/tag_count_reward/std": 0.18019695580005646, + "step": 2347 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2005.0, + "completions/mean_length": 836.697265625, + "completions/mean_terminated_length": 792.5607299804688, + "completions/min_length": 256.0, + "completions/min_terminated_length": 256.0, + "epoch": 0.8015703678415976, + "grad_norm": 0.9718857407569885, + "kl": 5.6875, + "learning_rate": 2.0390550572243242e-07, + "loss": 0.3645, + "num_tokens": 1264292545.0, + "reward": 1.89111328125, + "reward_std": 0.4370141923427582, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.890625, + "rewards/format_reward/std": 0.31241437792778015, + "rewards/tag_count_reward/mean": 0.93798828125, + "rewards/tag_count_reward/std": 0.18422965705394745, + "step": 2348 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 874.984375, + "completions/mean_terminated_length": 842.0079956054688, + "completions/min_length": 171.0, + "completions/min_terminated_length": 171.0, + "epoch": 0.8019117521549884, + "grad_norm": 1.066348910331726, + "kl": 5.609375, + "learning_rate": 2.0356310883097045e-07, + "loss": 0.3583, + "num_tokens": 1264819481.0, + "reward": 1.90185546875, + "reward_std": 0.4515777826309204, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.896484375, + "rewards/format_reward/std": 0.30492907762527466, + "rewards/tag_count_reward/mean": 0.94482421875, + "rewards/tag_count_reward/std": 0.16919739544391632, + "step": 2349 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1907.0, + "completions/mean_length": 794.236328125, + "completions/mean_terminated_length": 751.1777954101562, + "completions/min_length": 147.0, + "completions/min_terminated_length": 147.0, + "epoch": 0.8022531364683793, + "grad_norm": 2.048433303833008, + "kl": 5.1171875, + "learning_rate": 2.0322120364331119e-07, + "loss": 0.3373, + "num_tokens": 1265299522.0, + "reward": 1.91845703125, + "reward_std": 0.4585955739021301, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.896484375, + "rewards/format_reward/std": 0.30492907762527466, + "rewards/tag_count_reward/mean": 0.94189453125, + "rewards/tag_count_reward/std": 0.1794690042734146, + "step": 2350 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2006.0, + "completions/mean_length": 838.75, + "completions/mean_terminated_length": 802.2534790039062, + "completions/min_length": 83.0, + "completions/min_terminated_length": 83.0, + "epoch": 0.8025945207817701, + "grad_norm": 4.7970685958862305, + "kl": 8.3046875, + "learning_rate": 2.028797906447268e-07, + "loss": 0.4665, + "num_tokens": 1265814274.0, + "reward": 1.85400390625, + "reward_std": 0.5632809400558472, + "rewards/accuracy_reward/mean": 0.1015625, + "rewards/accuracy_reward/std": 0.30236753821372986, + "rewards/format_reward/mean": 0.830078125, + "rewards/format_reward/std": 0.3759314715862274, + "rewards/tag_count_reward/mean": 0.92236328125, + "rewards/tag_count_reward/std": 0.19585449993610382, + "step": 2351 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1941.0, + "completions/mean_length": 829.587890625, + "completions/mean_terminated_length": 782.6307983398438, + "completions/min_length": 134.0, + "completions/min_terminated_length": 134.0, + "epoch": 0.8029359050951609, + "grad_norm": 2.448554277420044, + "kl": 6.640625, + "learning_rate": 2.025388703197903e-07, + "loss": 0.4144, + "num_tokens": 1266320575.0, + "reward": 1.95458984375, + "reward_std": 0.503436803817749, + "rewards/accuracy_reward/mean": 0.12298387289047241, + "rewards/accuracy_reward/std": 0.32875028252601624, + "rewards/format_reward/mean": 0.888671875, + "rewards/format_reward/std": 0.31484565138816833, + "rewards/tag_count_reward/mean": 0.94677734375, + "rewards/tag_count_reward/std": 0.16910135746002197, + "step": 2352 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1999.0, + "completions/mean_length": 842.962890625, + "completions/mean_terminated_length": 809.0863037109375, + "completions/min_length": 131.0, + "completions/min_terminated_length": 131.0, + "epoch": 0.8032772894085517, + "grad_norm": 3.1435487270355225, + "kl": 7.37109375, + "learning_rate": 2.0219844315237595e-07, + "loss": 0.4264, + "num_tokens": 1266832396.0, + "reward": 1.88037109375, + "reward_std": 0.5150898694992065, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.92919921875, + "rewards/tag_count_reward/std": 0.1915329545736313, + "step": 2353 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2013.0, + "completions/mean_length": 875.8984375, + "completions/mean_terminated_length": 830.7261352539062, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "epoch": 0.8036186737219425, + "grad_norm": 3.2759320735931396, + "kl": 8.3671875, + "learning_rate": 2.0185850962565782e-07, + "loss": 0.5176, + "num_tokens": 1267364200.0, + "reward": 1.90869140625, + "reward_std": 0.540012001991272, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.93408203125, + "rewards/tag_count_reward/std": 0.1932711899280548, + "step": 2354 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2002.0, + "completions/mean_length": 832.54296875, + "completions/mean_terminated_length": 805.8562622070312, + "completions/min_length": 127.0, + "completions/min_terminated_length": 127.0, + "epoch": 0.8039600580353333, + "grad_norm": 1.754936933517456, + "kl": 4.85546875, + "learning_rate": 2.0151907022210917e-07, + "loss": 0.3045, + "num_tokens": 1267873438.0, + "reward": 1.91552734375, + "reward_std": 0.4817492365837097, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.888671875, + "rewards/format_reward/std": 0.31484565138816833, + "rewards/tag_count_reward/mean": 0.94287109375, + "rewards/tag_count_reward/std": 0.17070980370044708, + "step": 2355 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1921.0, + "completions/mean_length": 756.267578125, + "completions/mean_terminated_length": 709.200439453125, + "completions/min_length": 75.0, + "completions/min_terminated_length": 75.0, + "epoch": 0.804301442348724, + "grad_norm": 1.607111930847168, + "kl": 6.8125, + "learning_rate": 2.0118012542350245e-07, + "loss": 0.4416, + "num_tokens": 1268334135.0, + "reward": 1.97607421875, + "reward_std": 0.5530889630317688, + "rewards/accuracy_reward/mean": 0.16015625, + "rewards/accuracy_reward/std": 0.3671095669269562, + "rewards/format_reward/mean": 0.876953125, + "rewards/format_reward/std": 0.32881227135658264, + "rewards/tag_count_reward/mean": 0.93896484375, + "rewards/tag_count_reward/std": 0.1798572540283203, + "step": 2356 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2001.0, + "completions/mean_length": 842.298828125, + "completions/mean_terminated_length": 798.3663940429688, + "completions/min_length": 125.0, + "completions/min_terminated_length": 125.0, + "epoch": 0.8046428266621148, + "grad_norm": 1.7854558229446411, + "kl": 6.0625, + "learning_rate": 2.0084167571090753e-07, + "loss": 0.3664, + "num_tokens": 1268849936.0, + "reward": 1.86474609375, + "reward_std": 0.5049247741699219, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.857421875, + "rewards/format_reward/std": 0.3499840497970581, + "rewards/tag_count_reward/mean": 0.92529296875, + "rewards/tag_count_reward/std": 0.1900404542684555, + "step": 2357 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1896.0, + "completions/mean_length": 905.337890625, + "completions/mean_terminated_length": 870.85107421875, + "completions/min_length": 131.0, + "completions/min_terminated_length": 131.0, + "epoch": 0.8049842109755057, + "grad_norm": 2.130006790161133, + "kl": 4.6015625, + "learning_rate": 2.0050372156469214e-07, + "loss": 0.2799, + "num_tokens": 1269401965.0, + "reward": 1.88232421875, + "reward_std": 0.49413174390792847, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.880859375, + "rewards/format_reward/std": 0.32427072525024414, + "rewards/tag_count_reward/mean": 0.94091796875, + "rewards/tag_count_reward/std": 0.1750049889087677, + "step": 2358 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 845.869140625, + "completions/mean_terminated_length": 794.4542236328125, + "completions/min_length": 143.0, + "completions/min_terminated_length": 143.0, + "epoch": 0.8053255952888965, + "grad_norm": 1.481907844543457, + "kl": 5.859375, + "learning_rate": 2.0016626346452027e-07, + "loss": 0.39, + "num_tokens": 1269916570.0, + "reward": 1.8525390625, + "reward_std": 0.41104599833488464, + "rewards/accuracy_reward/mean": 0.01953125, + "rewards/accuracy_reward/std": 0.1385180652141571, + "rewards/format_reward/mean": 0.892578125, + "rewards/format_reward/std": 0.30995169281959534, + "rewards/tag_count_reward/mean": 0.9404296875, + "rewards/tag_count_reward/std": 0.17518849670886993, + "step": 2359 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2025.0, + "completions/mean_length": 792.029296875, + "completions/mean_terminated_length": 751.5140991210938, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "epoch": 0.8056669796022873, + "grad_norm": 1.5508971214294434, + "kl": 5.56640625, + "learning_rate": 1.998293018893518e-07, + "loss": 0.3609, + "num_tokens": 1270392025.0, + "reward": 1.89794921875, + "reward_std": 0.4793471395969391, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.880859375, + "rewards/format_reward/std": 0.32427072525024414, + "rewards/tag_count_reward/mean": 0.94091796875, + "rewards/tag_count_reward/std": 0.17570249736309052, + "step": 2360 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1950.0, + "completions/mean_length": 796.81640625, + "completions/mean_terminated_length": 771.8924560546875, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "epoch": 0.8060083639156781, + "grad_norm": 1.2236746549606323, + "kl": 4.984375, + "learning_rate": 1.994928373174423e-07, + "loss": 0.3234, + "num_tokens": 1270871115.0, + "reward": 1.9189453125, + "reward_std": 0.5018002986907959, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.880859375, + "rewards/format_reward/std": 0.32427072525024414, + "rewards/tag_count_reward/mean": 0.9443359375, + "rewards/tag_count_reward/std": 0.16867490112781525, + "step": 2361 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1669.0, + "completions/mean_length": 734.2421875, + "completions/mean_terminated_length": 708.0717163085938, + "completions/min_length": 135.0, + "completions/min_terminated_length": 135.0, + "epoch": 0.8063497482290689, + "grad_norm": 1.1561851501464844, + "kl": 3.3671875, + "learning_rate": 1.991568702263415e-07, + "loss": 0.203, + "num_tokens": 1271328935.0, + "reward": 2.00146484375, + "reward_std": 0.4283190071582794, + "rewards/accuracy_reward/mean": 0.1328125, + "rewards/accuracy_reward/std": 0.33970388770103455, + "rewards/format_reward/mean": 0.912109375, + "rewards/format_reward/std": 0.2834126651287079, + "rewards/tag_count_reward/mean": 0.95654296875, + "rewards/tag_count_reward/std": 0.153054878115654, + "step": 2362 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 866.05078125, + "completions/mean_terminated_length": 807.9220581054688, + "completions/min_length": 183.0, + "completions/min_terminated_length": 183.0, + "epoch": 0.8066911325424597, + "grad_norm": 1.1642218828201294, + "kl": 6.8359375, + "learning_rate": 1.9882140109289342e-07, + "loss": 0.4629, + "num_tokens": 1271850433.0, + "reward": 1.8515625, + "reward_std": 0.5475321412086487, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.927734375, + "rewards/tag_count_reward/std": 0.19384446740150452, + "step": 2363 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1956.0, + "completions/mean_length": 813.064453125, + "completions/mean_terminated_length": 762.86376953125, + "completions/min_length": 159.0, + "completions/min_terminated_length": 159.0, + "epoch": 0.8070325168558504, + "grad_norm": 1.0698622465133667, + "kl": 6.3046875, + "learning_rate": 1.9848643039323499e-07, + "loss": 0.3831, + "num_tokens": 1272336914.0, + "reward": 1.9189453125, + "reward_std": 0.5730538368225098, + "rewards/accuracy_reward/mean": 0.126953125, + "rewards/accuracy_reward/std": 0.33324605226516724, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.9267578125, + "rewards/tag_count_reward/std": 0.19908507168293, + "step": 2364 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1850.0, + "completions/mean_length": 817.259765625, + "completions/mean_terminated_length": 774.991943359375, + "completions/min_length": 89.0, + "completions/min_terminated_length": 89.0, + "epoch": 0.8073739011692412, + "grad_norm": 1.8406894207000732, + "kl": 6.78125, + "learning_rate": 1.9815195860279594e-07, + "loss": 0.4044, + "num_tokens": 1272834247.0, + "reward": 1.8310546875, + "reward_std": 0.5293779373168945, + "rewards/accuracy_reward/mean": 0.0463709682226181, + "rewards/accuracy_reward/std": 0.21049949526786804, + "rewards/format_reward/mean": 0.859375, + "rewards/format_reward/std": 0.3479743003845215, + "rewards/tag_count_reward/mean": 0.9267578125, + "rewards/tag_count_reward/std": 0.19908507168293, + "step": 2365 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2016.0, + "completions/mean_length": 853.341796875, + "completions/mean_terminated_length": 814.8043823242188, + "completions/min_length": 129.0, + "completions/min_terminated_length": 129.0, + "epoch": 0.807715285482632, + "grad_norm": 1.2284071445465088, + "kl": 4.6171875, + "learning_rate": 1.978179861962978e-07, + "loss": 0.2866, + "num_tokens": 1273352966.0, + "reward": 1.93408203125, + "reward_std": 0.4858850836753845, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.888671875, + "rewards/format_reward/std": 0.31484565138816833, + "rewards/tag_count_reward/mean": 0.94970703125, + "rewards/tag_count_reward/std": 0.15883907675743103, + "step": 2366 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1987.0, + "completions/mean_length": 804.623046875, + "completions/mean_terminated_length": 764.5140991210938, + "completions/min_length": 183.0, + "completions/min_terminated_length": 183.0, + "epoch": 0.8080566697960229, + "grad_norm": 1.5557031631469727, + "kl": 6.0859375, + "learning_rate": 1.9748451364775318e-07, + "loss": 0.3996, + "num_tokens": 1273850533.0, + "reward": 1.84814453125, + "reward_std": 0.47291260957717896, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.873046875, + "rewards/format_reward/std": 0.33324605226516724, + "rewards/tag_count_reward/mean": 0.93408203125, + "rewards/tag_count_reward/std": 0.1788075864315033, + "step": 2367 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2030.0, + "completions/mean_length": 784.146484375, + "completions/mean_terminated_length": 764.0853881835938, + "completions/min_length": 172.0, + "completions/min_terminated_length": 172.0, + "epoch": 0.8083980541094137, + "grad_norm": 1.4546853303909302, + "kl": 4.65625, + "learning_rate": 1.9715154143046558e-07, + "loss": 0.2798, + "num_tokens": 1274335216.0, + "reward": 1.88720703125, + "reward_std": 0.4807460308074951, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.876953125, + "rewards/format_reward/std": 0.32881227135658264, + "rewards/tag_count_reward/mean": 0.93994140625, + "rewards/tag_count_reward/std": 0.17396998405456543, + "step": 2368 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1973.0, + "completions/mean_length": 800.859375, + "completions/mean_terminated_length": 765.7991943359375, + "completions/min_length": 146.0, + "completions/min_terminated_length": 146.0, + "epoch": 0.8087394384228045, + "grad_norm": 1.8968936204910278, + "kl": 5.03125, + "learning_rate": 1.968190700170279e-07, + "loss": 0.3277, + "num_tokens": 1274820872.0, + "reward": 1.97900390625, + "reward_std": 0.5076891183853149, + "rewards/accuracy_reward/mean": 0.142578125, + "rewards/accuracy_reward/std": 0.3499840497970581, + "rewards/format_reward/mean": 0.892578125, + "rewards/format_reward/std": 0.30995169281959534, + "rewards/tag_count_reward/mean": 0.94384765625, + "rewards/tag_count_reward/std": 0.17174777388572693, + "step": 2369 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 766.546875, + "completions/mean_terminated_length": 741.0199584960938, + "completions/min_length": 119.0, + "completions/min_terminated_length": 119.0, + "epoch": 0.8090808227361953, + "grad_norm": 1.373109221458435, + "kl": 4.734375, + "learning_rate": 1.9648709987932282e-07, + "loss": 0.315, + "num_tokens": 1275286736.0, + "reward": 1.8740234375, + "reward_std": 0.4075538218021393, + "rewards/accuracy_reward/mean": 0.037109375, + "rewards/accuracy_reward/std": 0.18921469151973724, + "rewards/format_reward/mean": 0.892578125, + "rewards/format_reward/std": 0.30995169281959534, + "rewards/tag_count_reward/mean": 0.9443359375, + "rewards/tag_count_reward/std": 0.17083640396595, + "step": 2370 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2033.0, + "completions/mean_length": 822.357421875, + "completions/mean_terminated_length": 785.3661499023438, + "completions/min_length": 113.0, + "completions/min_terminated_length": 113.0, + "epoch": 0.8094222070495861, + "grad_norm": 2.1128485202789307, + "kl": 5.21875, + "learning_rate": 1.9615563148852092e-07, + "loss": 0.3068, + "num_tokens": 1275800407.0, + "reward": 1.89599609375, + "reward_std": 0.4957549571990967, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.8828125, + "rewards/format_reward/std": 0.32195815443992615, + "rewards/tag_count_reward/mean": 0.93896484375, + "rewards/tag_count_reward/std": 0.17917592823505402, + "step": 2371 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1932.0, + "completions/mean_length": 853.7734375, + "completions/mean_terminated_length": 802.6965942382812, + "completions/min_length": 143.0, + "completions/min_terminated_length": 143.0, + "epoch": 0.8097635913629768, + "grad_norm": 2.7275819778442383, + "kl": 7.1796875, + "learning_rate": 1.9582466531508135e-07, + "loss": 0.4194, + "num_tokens": 1276318851.0, + "reward": 1.8271484375, + "reward_std": 0.48342227935791016, + "rewards/accuracy_reward/mean": 0.03125, + "rewards/accuracy_reward/std": 0.17416280508041382, + "rewards/format_reward/mean": 0.86328125, + "rewards/format_reward/std": 0.3438861668109894, + "rewards/tag_count_reward/mean": 0.9326171875, + "rewards/tag_count_reward/std": 0.18266506493091583, + "step": 2372 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 834.15625, + "completions/mean_terminated_length": 805.0240478515625, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "epoch": 0.8101049756763676, + "grad_norm": 1.0214875936508179, + "kl": 5.5859375, + "learning_rate": 1.9549420182874956e-07, + "loss": 0.3217, + "num_tokens": 1276828035.0, + "reward": 1.89794921875, + "reward_std": 0.4587544798851013, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.8984375, + "rewards/format_reward/std": 0.30236753821372986, + "rewards/tag_count_reward/mean": 0.94873046875, + "rewards/tag_count_reward/std": 0.17114026844501495, + "step": 2373 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2016.0, + "completions/mean_length": 834.70703125, + "completions/mean_terminated_length": 782.814697265625, + "completions/min_length": 52.0, + "completions/min_terminated_length": 52.0, + "epoch": 0.8104463599897584, + "grad_norm": 1.7794749736785889, + "kl": 6.98828125, + "learning_rate": 1.9516424149855829e-07, + "loss": 0.4101, + "num_tokens": 1277335549.0, + "reward": 1.78564453125, + "reward_std": 0.525837242603302, + "rewards/accuracy_reward/mean": 0.0234375, + "rewards/accuracy_reward/std": 0.15143637359142303, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.91845703125, + "rewards/tag_count_reward/std": 0.2028808295726776, + "step": 2374 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1995.0, + "completions/mean_length": 886.5390625, + "completions/mean_terminated_length": 846.6505737304688, + "completions/min_length": 112.0, + "completions/min_terminated_length": 112.0, + "epoch": 0.8107877443031493, + "grad_norm": 1.9440349340438843, + "kl": 6.9609375, + "learning_rate": 1.94834784792826e-07, + "loss": 0.4378, + "num_tokens": 1277865841.0, + "reward": 1.88330078125, + "reward_std": 0.5041627883911133, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.92822265625, + "rewards/tag_count_reward/std": 0.20053589344024658, + "step": 2375 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 861.6484375, + "completions/mean_terminated_length": 803.30322265625, + "completions/min_length": 35.0, + "completions/min_terminated_length": 35.0, + "epoch": 0.8111291286165401, + "grad_norm": 1.0282351970672607, + "kl": 8.3515625, + "learning_rate": 1.9450583217915595e-07, + "loss": 0.5511, + "num_tokens": 1278386477.0, + "reward": 1.81591796875, + "reward_std": 0.5656579732894897, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.828125, + "rewards/format_reward/std": 0.3776407241821289, + "rewards/tag_count_reward/mean": 0.91162109375, + "rewards/tag_count_reward/std": 0.21643958985805511, + "step": 2376 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2006.0, + "completions/mean_length": 825.478515625, + "completions/mean_terminated_length": 767.9774780273438, + "completions/min_length": 5.0, + "completions/min_terminated_length": 5.0, + "epoch": 0.8114705129299309, + "grad_norm": 1.0348620414733887, + "kl": 6.08984375, + "learning_rate": 1.9417738412443647e-07, + "loss": 0.3737, + "num_tokens": 1278889826.0, + "reward": 1.89501953125, + "reward_std": 0.5118416547775269, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.8671875, + "rewards/format_reward/std": 0.33970388770103455, + "rewards/tag_count_reward/mean": 0.93212890625, + "rewards/tag_count_reward/std": 0.19511640071868896, + "step": 2377 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 824.9453125, + "completions/mean_terminated_length": 790.5621948242188, + "completions/min_length": 186.0, + "completions/min_terminated_length": 186.0, + "epoch": 0.8118118972433217, + "grad_norm": 1.0888521671295166, + "kl": 5.9453125, + "learning_rate": 1.938494410948394e-07, + "loss": 0.3837, + "num_tokens": 1279386534.0, + "reward": 1.8447265625, + "reward_std": 0.4803759455680847, + "rewards/accuracy_reward/mean": 0.037109375, + "rewards/accuracy_reward/std": 0.18921469151973724, + "rewards/format_reward/mean": 0.873046875, + "rewards/format_reward/std": 0.33324605226516724, + "rewards/tag_count_reward/mean": 0.9345703125, + "rewards/tag_count_reward/std": 0.18992790579795837, + "step": 2378 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2018.0, + "completions/mean_length": 778.08203125, + "completions/mean_terminated_length": 747.6040649414062, + "completions/min_length": 169.0, + "completions/min_terminated_length": 169.0, + "epoch": 0.8121532815567125, + "grad_norm": 1.3943076133728027, + "kl": 5.34375, + "learning_rate": 1.9352200355581988e-07, + "loss": 0.3171, + "num_tokens": 1279861152.0, + "reward": 1.9248046875, + "reward_std": 0.4652273654937744, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.890625, + "rewards/format_reward/std": 0.31241437792778015, + "rewards/tag_count_reward/mean": 0.9423828125, + "rewards/tag_count_reward/std": 0.1723288893699646, + "step": 2379 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 828.548828125, + "completions/mean_terminated_length": 791.7444458007812, + "completions/min_length": 240.0, + "completions/min_terminated_length": 240.0, + "epoch": 0.8124946658701032, + "grad_norm": 1.807580828666687, + "kl": 6.75, + "learning_rate": 1.9319507197211538e-07, + "loss": 0.445, + "num_tokens": 1280359017.0, + "reward": 1.859375, + "reward_std": 0.5383387804031372, + "rewards/accuracy_reward/mean": 0.08669354766607285, + "rewards/accuracy_reward/std": 0.281669557094574, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.921875, + "rewards/tag_count_reward/std": 0.2009030431509018, + "step": 2380 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1957.0, + "completions/mean_length": 798.341796875, + "completions/mean_terminated_length": 765.7855834960938, + "completions/min_length": 188.0, + "completions/min_terminated_length": 188.0, + "epoch": 0.812836050183494, + "grad_norm": 2.365671396255493, + "kl": 5.4453125, + "learning_rate": 1.9286864680774578e-07, + "loss": 0.3664, + "num_tokens": 1280844168.0, + "reward": 1.876953125, + "reward_std": 0.4589555859565735, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.896484375, + "rewards/format_reward/std": 0.30492907762527466, + "rewards/tag_count_reward/mean": 0.9375, + "rewards/tag_count_reward/std": 0.17832663655281067, + "step": 2381 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1962.0, + "completions/mean_length": 853.376953125, + "completions/mean_terminated_length": 822.2545166015625, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "epoch": 0.8131774344968848, + "grad_norm": 1.2380436658859253, + "kl": 5.953125, + "learning_rate": 1.9254272852601193e-07, + "loss": 0.4262, + "num_tokens": 1281364601.0, + "reward": 1.86865234375, + "reward_std": 0.4679795801639557, + "rewards/accuracy_reward/mean": 0.04435483738780022, + "rewards/accuracy_reward/std": 0.2060900777578354, + "rewards/format_reward/mean": 0.884765625, + "rewards/format_reward/std": 0.3196168541908264, + "rewards/tag_count_reward/mean": 0.94091796875, + "rewards/tag_count_reward/std": 0.17914927005767822, + "step": 2382 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1992.0, + "completions/mean_length": 691.244140625, + "completions/mean_terminated_length": 672.4376220703125, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "epoch": 0.8135188188102757, + "grad_norm": 1.8697482347488403, + "kl": 4.03515625, + "learning_rate": 1.9221731758949498e-07, + "loss": 0.3017, + "num_tokens": 1281791078.0, + "reward": 2.037109375, + "reward_std": 0.45592376589775085, + "rewards/accuracy_reward/mean": 0.150390625, + "rewards/accuracy_reward/std": 0.35780346393585205, + "rewards/format_reward/mean": 0.92578125, + "rewards/format_reward/std": 0.2623828947544098, + "rewards/tag_count_reward/mean": 0.9609375, + "rewards/tag_count_reward/std": 0.1465102881193161, + "step": 2383 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1913.0, + "completions/mean_length": 785.208984375, + "completions/mean_terminated_length": 747.0965576171875, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "epoch": 0.8138602031236665, + "grad_norm": 0.859073281288147, + "kl": 6.3984375, + "learning_rate": 1.918924144600566e-07, + "loss": 0.4384, + "num_tokens": 1282270705.0, + "reward": 1.91259765625, + "reward_std": 0.47794193029403687, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.892578125, + "rewards/format_reward/std": 0.30995169281959534, + "rewards/tag_count_reward/mean": 0.94384765625, + "rewards/tag_count_reward/std": 0.1659528613090515, + "step": 2384 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1987.0, + "completions/mean_length": 847.498046875, + "completions/mean_terminated_length": 798.6971435546875, + "completions/min_length": 88.0, + "completions/min_terminated_length": 88.0, + "epoch": 0.8142015874370573, + "grad_norm": 2.2664906978607178, + "kl": 5.44921875, + "learning_rate": 1.915680195988369e-07, + "loss": 0.3745, + "num_tokens": 1282790496.0, + "reward": 1.89892578125, + "reward_std": 0.4867627024650574, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.888671875, + "rewards/format_reward/std": 0.31484565138816833, + "rewards/tag_count_reward/mean": 0.94189453125, + "rewards/tag_count_reward/std": 0.17533229291439056, + "step": 2385 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1966.0, + "completions/mean_length": 824.65234375, + "completions/mean_terminated_length": 800.2828979492188, + "completions/min_length": 19.0, + "completions/min_terminated_length": 19.0, + "epoch": 0.8145429717504481, + "grad_norm": 1.8327305316925049, + "kl": 4.3828125, + "learning_rate": 1.912441334662554e-07, + "loss": 0.296, + "num_tokens": 1283294270.0, + "reward": 1.8974609375, + "reward_std": 0.38229382038116455, + "rewards/accuracy_reward/mean": 0.03125, + "rewards/accuracy_reward/std": 0.17416280508041382, + "rewards/format_reward/mean": 0.9140625, + "rewards/format_reward/std": 0.28054583072662354, + "rewards/tag_count_reward/mean": 0.9521484375, + "rewards/tag_count_reward/std": 0.15453127026557922, + "step": 2386 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1902.0, + "completions/mean_length": 763.08984375, + "completions/mean_terminated_length": 726.9678344726562, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "epoch": 0.8148843560638389, + "grad_norm": 1.331518292427063, + "kl": 5.4609375, + "learning_rate": 1.9092075652200894e-07, + "loss": 0.3553, + "num_tokens": 1283764812.0, + "reward": 1.92724609375, + "reward_std": 0.44393885135650635, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.904296875, + "rewards/format_reward/std": 0.2944713830947876, + "rewards/tag_count_reward/mean": 0.95068359375, + "rewards/tag_count_reward/std": 0.15914559364318848, + "step": 2387 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 739.494140625, + "completions/mean_terminated_length": 694.5556030273438, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "epoch": 0.8152257403772296, + "grad_norm": 1.7126271724700928, + "kl": 7.109375, + "learning_rate": 1.9059788922507213e-07, + "loss": 0.466, + "num_tokens": 1284229129.0, + "reward": 1.88134765625, + "reward_std": 0.47255924344062805, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.896484375, + "rewards/format_reward/std": 0.30492907762527466, + "rewards/tag_count_reward/mean": 0.94189453125, + "rewards/tag_count_reward/std": 0.1794690042734146, + "step": 2388 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 827.318359375, + "completions/mean_terminated_length": 795.5170288085938, + "completions/min_length": 156.0, + "completions/min_terminated_length": 156.0, + "epoch": 0.8155671246906204, + "grad_norm": 3.0533907413482666, + "kl": 6.9296875, + "learning_rate": 1.902755320336961e-07, + "loss": 0.4197, + "num_tokens": 1284730172.0, + "reward": 1.88134765625, + "reward_std": 0.45382267236709595, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.884765625, + "rewards/format_reward/std": 0.3196168541908264, + "rewards/tag_count_reward/mean": 0.94189453125, + "rewards/tag_count_reward/std": 0.18082687258720398, + "step": 2389 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 828.9609375, + "completions/mean_terminated_length": 789.6370849609375, + "completions/min_length": 160.0, + "completions/min_terminated_length": 160.0, + "epoch": 0.8159085090040112, + "grad_norm": 3.313844680786133, + "kl": 8.3515625, + "learning_rate": 1.899536854054079e-07, + "loss": 0.5099, + "num_tokens": 1285232248.0, + "reward": 1.8818359375, + "reward_std": 0.4815102517604828, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.88671875, + "rewards/format_reward/std": 0.3172462284564972, + "rewards/tag_count_reward/mean": 0.9345703125, + "rewards/tag_count_reward/std": 0.19121153652668, + "step": 2390 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1992.0, + "completions/mean_length": 807.07421875, + "completions/mean_terminated_length": 767.0443115234375, + "completions/min_length": 57.0, + "completions/min_terminated_length": 57.0, + "epoch": 0.816249893317402, + "grad_norm": 3.641836404800415, + "kl": 8.265625, + "learning_rate": 1.8963234979700986e-07, + "loss": 0.4863, + "num_tokens": 1285725182.0, + "reward": 1.8818359375, + "reward_std": 0.5220977663993835, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.876953125, + "rewards/format_reward/std": 0.32881227135658264, + "rewards/tag_count_reward/mean": 0.9267578125, + "rewards/tag_count_reward/std": 0.2033400982618332, + "step": 2391 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1901.0, + "completions/mean_length": 738.0390625, + "completions/mean_terminated_length": 698.5029907226562, + "completions/min_length": 155.0, + "completions/min_terminated_length": 155.0, + "epoch": 0.8165912776307929, + "grad_norm": 1.327378273010254, + "kl": 7.3359375, + "learning_rate": 1.8931152566457903e-07, + "loss": 0.4818, + "num_tokens": 1286178706.0, + "reward": 1.951171875, + "reward_std": 0.5315266847610474, + "rewards/accuracy_reward/mean": 0.1171875, + "rewards/accuracy_reward/std": 0.32195815443992615, + "rewards/format_reward/mean": 0.890625, + "rewards/format_reward/std": 0.31241437792778015, + "rewards/tag_count_reward/mean": 0.943359375, + "rewards/tag_count_reward/std": 0.1712302714586258, + "step": 2392 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2024.0, + "completions/mean_length": 775.56640625, + "completions/mean_terminated_length": 731.86669921875, + "completions/min_length": 158.0, + "completions/min_terminated_length": 158.0, + "epoch": 0.8169326619441837, + "grad_norm": 2.8124125003814697, + "kl": 7.0390625, + "learning_rate": 1.8899121346346682e-07, + "loss": 0.4314, + "num_tokens": 1286650340.0, + "reward": 1.88525390625, + "reward_std": 0.4600207805633545, + "rewards/accuracy_reward/mean": 0.058467742055654526, + "rewards/accuracy_reward/std": 0.23486268520355225, + "rewards/format_reward/mean": 0.890625, + "rewards/format_reward/std": 0.31241437792778015, + "rewards/tag_count_reward/mean": 0.93798828125, + "rewards/tag_count_reward/std": 0.18222707509994507, + "step": 2393 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1967.0, + "completions/mean_length": 825.013671875, + "completions/mean_terminated_length": 785.5625, + "completions/min_length": 108.0, + "completions/min_terminated_length": 108.0, + "epoch": 0.8172740462575745, + "grad_norm": 1.3582987785339355, + "kl": 6.0390625, + "learning_rate": 1.8867141364829758e-07, + "loss": 0.3957, + "num_tokens": 1287145483.0, + "reward": 1.9453125, + "reward_std": 0.43716537952423096, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.912109375, + "rewards/format_reward/std": 0.2834126651287079, + "rewards/tag_count_reward/mean": 0.951171875, + "rewards/tag_count_reward/std": 0.16495446860790253, + "step": 2394 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2006.0, + "completions/mean_length": 801.60546875, + "completions/mean_terminated_length": 753.5699462890625, + "completions/min_length": 118.0, + "completions/min_terminated_length": 118.0, + "epoch": 0.8176154305709653, + "grad_norm": 1.6503441333770752, + "kl": 6.921875, + "learning_rate": 1.8835212667296873e-07, + "loss": 0.434, + "num_tokens": 1287640433.0, + "reward": 1.86767578125, + "reward_std": 0.42626187205314636, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.884765625, + "rewards/format_reward/std": 0.3196168541908264, + "rewards/tag_count_reward/mean": 0.93603515625, + "rewards/tag_count_reward/std": 0.18751464784145355, + "step": 2395 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1977.0, + "completions/mean_length": 781.658203125, + "completions/mean_terminated_length": 743.4385986328125, + "completions/min_length": 96.0, + "completions/min_terminated_length": 96.0, + "epoch": 0.817956814884356, + "grad_norm": 0.7858288288116455, + "kl": 6.015625, + "learning_rate": 1.8803335299064998e-07, + "loss": 0.395, + "num_tokens": 1288118450.0, + "reward": 1.8798828125, + "reward_std": 0.48168665170669556, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.876953125, + "rewards/format_reward/std": 0.32881227135658264, + "rewards/tag_count_reward/mean": 0.9345703125, + "rewards/tag_count_reward/std": 0.18536527454853058, + "step": 2396 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1963.0, + "completions/mean_length": 895.197265625, + "completions/mean_terminated_length": 836.0184936523438, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "epoch": 0.8182981991977468, + "grad_norm": 1.5382969379425049, + "kl": 7.4140625, + "learning_rate": 1.8771509305378186e-07, + "loss": 0.4673, + "num_tokens": 1288652135.0, + "reward": 1.7978515625, + "reward_std": 0.5393427014350891, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.9189453125, + "rewards/tag_count_reward/std": 0.20277541875839233, + "step": 2397 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1920.0, + "completions/mean_length": 825.15234375, + "completions/mean_terminated_length": 772.8513793945312, + "completions/min_length": 5.0, + "completions/min_terminated_length": 5.0, + "epoch": 0.8186395835111376, + "grad_norm": 1.5090433359146118, + "kl": 4.8515625, + "learning_rate": 1.8739734731407646e-07, + "loss": 0.3283, + "num_tokens": 1289155061.0, + "reward": 1.888671875, + "reward_std": 0.46926945447921753, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.888671875, + "rewards/format_reward/std": 0.31484565138816833, + "rewards/tag_count_reward/mean": 0.935546875, + "rewards/tag_count_reward/std": 0.18702034652233124, + "step": 2398 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1988.0, + "completions/mean_length": 781.5, + "completions/mean_terminated_length": 743.275634765625, + "completions/min_length": 63.0, + "completions/min_terminated_length": 63.0, + "epoch": 0.8189809678245284, + "grad_norm": 1.3026745319366455, + "kl": 5.23046875, + "learning_rate": 1.870801162225155e-07, + "loss": 0.3318, + "num_tokens": 1289630501.0, + "reward": 1.923828125, + "reward_std": 0.4812939167022705, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.884765625, + "rewards/format_reward/std": 0.3196168541908264, + "rewards/tag_count_reward/mean": 0.9453125, + "rewards/tag_count_reward/std": 0.17468871176242828, + "step": 2399 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2005.0, + "completions/mean_length": 808.80078125, + "completions/mean_terminated_length": 758.4268188476562, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "epoch": 0.8193223521379193, + "grad_norm": 1.4187021255493164, + "kl": 6.4609375, + "learning_rate": 1.8676340022935073e-07, + "loss": 0.4109, + "num_tokens": 1290125391.0, + "reward": 1.85498046875, + "reward_std": 0.4715687036514282, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.93505859375, + "rewards/tag_count_reward/std": 0.18454577028751373, + "step": 2400 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1971.0, + "completions/mean_length": 882.208984375, + "completions/mean_terminated_length": 817.309326171875, + "completions/min_length": 158.0, + "completions/min_terminated_length": 158.0, + "epoch": 0.8196637364513101, + "grad_norm": 1.629425048828125, + "kl": 6.328125, + "learning_rate": 1.8644719978410227e-07, + "loss": 0.4187, + "num_tokens": 1290648746.0, + "reward": 1.81005859375, + "reward_std": 0.46727508306503296, + "rewards/accuracy_reward/mean": 0.015625, + "rewards/accuracy_reward/std": 0.12414088100194931, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.92919921875, + "rewards/tag_count_reward/std": 0.198433056473732, + "step": 2401 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.060546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 880.744140625, + "completions/mean_terminated_length": 805.515625, + "completions/min_length": 148.0, + "completions/min_terminated_length": 148.0, + "epoch": 0.8200051207647009, + "grad_norm": 2.2330965995788574, + "kl": 7.90625, + "learning_rate": 1.861315153355592e-07, + "loss": 0.4897, + "num_tokens": 1291173639.0, + "reward": 1.82275390625, + "reward_std": 0.5710577964782715, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.91259765625, + "rewards/tag_count_reward/std": 0.2157054841518402, + "step": 2402 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2011.0, + "completions/mean_length": 806.13671875, + "completions/mean_terminated_length": 776.33203125, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "epoch": 0.8203465050780917, + "grad_norm": 1.571038007736206, + "kl": 5.5390625, + "learning_rate": 1.8581634733177758e-07, + "loss": 0.3696, + "num_tokens": 1291659277.0, + "reward": 1.8994140625, + "reward_std": 0.5044021606445312, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.880859375, + "rewards/format_reward/std": 0.32427072525024414, + "rewards/tag_count_reward/mean": 0.9443359375, + "rewards/tag_count_reward/std": 0.17083640396595, + "step": 2403 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1954.0, + "completions/mean_length": 880.21875, + "completions/mean_terminated_length": 797.15478515625, + "completions/min_length": 129.0, + "completions/min_terminated_length": 129.0, + "epoch": 0.8206878893914825, + "grad_norm": 1.3357652425765991, + "kl": 7.984375, + "learning_rate": 1.8550169622008078e-07, + "loss": 0.5268, + "num_tokens": 1292183805.0, + "reward": 1.81005859375, + "reward_std": 0.5667085647583008, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.91552734375, + "rewards/tag_count_reward/std": 0.21346116065979004, + "step": 2404 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1991.0, + "completions/mean_length": 781.201171875, + "completions/mean_terminated_length": 740.336669921875, + "completions/min_length": 139.0, + "completions/min_terminated_length": 139.0, + "epoch": 0.8210292737048732, + "grad_norm": 1.2438210248947144, + "kl": 5.265625, + "learning_rate": 1.851875624470586e-07, + "loss": 0.3265, + "num_tokens": 1292659876.0, + "reward": 1.8740234375, + "reward_std": 0.4153513014316559, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.8828125, + "rewards/format_reward/std": 0.32195815443992615, + "rewards/tag_count_reward/mean": 0.9443359375, + "rewards/tag_count_reward/std": 0.1693984717130661, + "step": 2405 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 890.201171875, + "completions/mean_terminated_length": 825.7463989257812, + "completions/min_length": 150.0, + "completions/min_terminated_length": 150.0, + "epoch": 0.821370658018264, + "grad_norm": 2.0447282791137695, + "kl": 7.4296875, + "learning_rate": 1.8487394645856636e-07, + "loss": 0.4506, + "num_tokens": 1293197355.0, + "reward": 1.82275390625, + "reward_std": 0.587771475315094, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.822265625, + "rewards/format_reward/std": 0.3826628625392914, + "rewards/tag_count_reward/mean": 0.91064453125, + "rewards/tag_count_reward/std": 0.21203739941120148, + "step": 2406 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1880.0, + "completions/mean_length": 772.44921875, + "completions/mean_terminated_length": 744.443115234375, + "completions/min_length": 101.0, + "completions/min_terminated_length": 101.0, + "epoch": 0.8217120423316548, + "grad_norm": 1.5338774919509888, + "kl": 4.79296875, + "learning_rate": 1.8456084869972472e-07, + "loss": 0.3286, + "num_tokens": 1293666113.0, + "reward": 1.916015625, + "reward_std": 0.4454042315483093, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.900390625, + "rewards/format_reward/std": 0.29977133870124817, + "rewards/tag_count_reward/mean": 0.947265625, + "rewards/tag_count_reward/std": 0.16224436461925507, + "step": 2407 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1954.0, + "completions/mean_length": 818.033203125, + "completions/mean_terminated_length": 778.3568115234375, + "completions/min_length": 157.0, + "completions/min_terminated_length": 157.0, + "epoch": 0.8220534266450457, + "grad_norm": 1.519821286201477, + "kl": 5.4453125, + "learning_rate": 1.8424826961491852e-07, + "loss": 0.3472, + "num_tokens": 1294152194.0, + "reward": 1.8525390625, + "reward_std": 0.4574953317642212, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.873046875, + "rewards/format_reward/std": 0.33324605226516724, + "rewards/tag_count_reward/mean": 0.9365234375, + "rewards/tag_count_reward/std": 0.18472495675086975, + "step": 2408 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1980.0, + "completions/mean_length": 853.595703125, + "completions/mean_terminated_length": 810.0748901367188, + "completions/min_length": 150.0, + "completions/min_terminated_length": 150.0, + "epoch": 0.8223948109584365, + "grad_norm": 1.4335675239562988, + "kl": 7.453125, + "learning_rate": 1.8393620964779675e-07, + "loss": 0.4401, + "num_tokens": 1294665027.0, + "reward": 1.7783203125, + "reward_std": 0.5727628469467163, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.80859375, + "rewards/format_reward/std": 0.3937928080558777, + "rewards/tag_count_reward/mean": 0.9052734375, + "rewards/tag_count_reward/std": 0.21798203885555267, + "step": 2409 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1886.0, + "completions/mean_length": 797.01171875, + "completions/mean_terminated_length": 756.6572265625, + "completions/min_length": 71.0, + "completions/min_terminated_length": 71.0, + "epoch": 0.8227361952718273, + "grad_norm": 1.3071527481079102, + "kl": 6.2421875, + "learning_rate": 1.8362466924127145e-07, + "loss": 0.3913, + "num_tokens": 1295146569.0, + "reward": 1.87890625, + "reward_std": 0.5147675275802612, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.935546875, + "rewards/tag_count_reward/std": 0.1850479692220688, + "step": 2410 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 778.109375, + "completions/mean_terminated_length": 726.48779296875, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "epoch": 0.8230775795852181, + "grad_norm": 2.4448471069335938, + "kl": 6.609375, + "learning_rate": 1.833136488375171e-07, + "loss": 0.4477, + "num_tokens": 1295620193.0, + "reward": 1.91357421875, + "reward_std": 0.5689276456832886, + "rewards/accuracy_reward/mean": 0.115234375, + "rewards/accuracy_reward/std": 0.3196168541908264, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.92919921875, + "rewards/tag_count_reward/std": 0.19280590116977692, + "step": 2411 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2008.0, + "completions/mean_length": 819.748046875, + "completions/mean_terminated_length": 792.7804565429688, + "completions/min_length": 183.0, + "completions/min_terminated_length": 183.0, + "epoch": 0.8234189638986089, + "grad_norm": 1.5323444604873657, + "kl": 5.390625, + "learning_rate": 1.8300314887797048e-07, + "loss": 0.3641, + "num_tokens": 1296110048.0, + "reward": 1.951171875, + "reward_std": 0.5348630547523499, + "rewards/accuracy_reward/mean": 0.12109375, + "rewards/accuracy_reward/std": 0.3265552520751953, + "rewards/format_reward/mean": 0.88671875, + "rewards/format_reward/std": 0.3172462284564972, + "rewards/tag_count_reward/mean": 0.943359375, + "rewards/tag_count_reward/std": 0.16762074828147888, + "step": 2412 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 809.5546875, + "completions/mean_terminated_length": 782.36328125, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "epoch": 0.8237603482119996, + "grad_norm": 1.4346827268600464, + "kl": 6.015625, + "learning_rate": 1.8269316980332926e-07, + "loss": 0.3519, + "num_tokens": 1296607004.0, + "reward": 1.87939453125, + "reward_std": 0.5118072032928467, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.86328125, + "rewards/format_reward/std": 0.3438861668109894, + "rewards/tag_count_reward/mean": 0.93408203125, + "rewards/tag_count_reward/std": 0.18617989122867584, + "step": 2413 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1959.0, + "completions/mean_length": 877.84375, + "completions/mean_terminated_length": 822.8057250976562, + "completions/min_length": 198.0, + "completions/min_terminated_length": 198.0, + "epoch": 0.8241017325253904, + "grad_norm": 2.058574914932251, + "kl": 8.0234375, + "learning_rate": 1.823837120535523e-07, + "loss": 0.4802, + "num_tokens": 1297139164.0, + "reward": 1.83935546875, + "reward_std": 0.5372190475463867, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.841796875, + "rewards/format_reward/std": 0.36528825759887695, + "rewards/tag_count_reward/mean": 0.91748046875, + "rewards/tag_count_reward/std": 0.2066698521375656, + "step": 2414 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2029.0, + "completions/mean_length": 830.666015625, + "completions/mean_terminated_length": 788.858642578125, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "epoch": 0.8244431168387812, + "grad_norm": 1.3971267938613892, + "kl": 6.703125, + "learning_rate": 1.820747760678581e-07, + "loss": 0.4367, + "num_tokens": 1297641553.0, + "reward": 1.8408203125, + "reward_std": 0.516970157623291, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.859375, + "rewards/format_reward/std": 0.3479743003845215, + "rewards/tag_count_reward/mean": 0.9306640625, + "rewards/tag_count_reward/std": 0.18723225593566895, + "step": 2415 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1988.0, + "completions/mean_length": 806.08984375, + "completions/mean_terminated_length": 791.3636474609375, + "completions/min_length": 4.0, + "completions/min_terminated_length": 4.0, + "epoch": 0.8247845011521721, + "grad_norm": 2.4153144359588623, + "kl": 3.859375, + "learning_rate": 1.8176636228472476e-07, + "loss": 0.2265, + "num_tokens": 1298131327.0, + "reward": 1.9189453125, + "reward_std": 0.43295690417289734, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.904296875, + "rewards/format_reward/std": 0.2944713830947876, + "rewards/tag_count_reward/mean": 0.9560546875, + "rewards/tag_count_reward/std": 0.14845077693462372, + "step": 2416 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1951.0, + "completions/mean_length": 811.267578125, + "completions/mean_terminated_length": 771.3729858398438, + "completions/min_length": 76.0, + "completions/min_terminated_length": 76.0, + "epoch": 0.8251258854655629, + "grad_norm": 1.9129817485809326, + "kl": 6.89453125, + "learning_rate": 1.814584711418894e-07, + "loss": 0.4248, + "num_tokens": 1298624424.0, + "reward": 1.830078125, + "reward_std": 0.5324984788894653, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.919921875, + "rewards/tag_count_reward/std": 0.2025608867406845, + "step": 2417 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1955.0, + "completions/mean_length": 868.361328125, + "completions/mean_terminated_length": 802.6907348632812, + "completions/min_length": 50.0, + "completions/min_terminated_length": 50.0, + "epoch": 0.8254672697789537, + "grad_norm": 3.6364946365356445, + "kl": 7.828125, + "learning_rate": 1.8115110307634695e-07, + "loss": 0.4445, + "num_tokens": 1299145841.0, + "reward": 1.86328125, + "reward_std": 0.5388745665550232, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.923828125, + "rewards/tag_count_reward/std": 0.199825257062912, + "step": 2418 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1950.0, + "completions/mean_length": 836.2890625, + "completions/mean_terminated_length": 807.2080688476562, + "completions/min_length": 183.0, + "completions/min_terminated_length": 183.0, + "epoch": 0.8258086540923445, + "grad_norm": 0.7928247451782227, + "kl": 5.90625, + "learning_rate": 1.8084425852435044e-07, + "loss": 0.3592, + "num_tokens": 1299650341.0, + "reward": 1.8720703125, + "reward_std": 0.49951568245887756, + "rewards/accuracy_reward/mean": 0.0729166641831398, + "rewards/accuracy_reward/std": 0.2602709233760834, + "rewards/format_reward/mean": 0.8671875, + "rewards/format_reward/std": 0.33970388770103455, + "rewards/tag_count_reward/mean": 0.9365234375, + "rewards/tag_count_reward/std": 0.17934982478618622, + "step": 2419 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1972.0, + "completions/mean_length": 750.4609375, + "completions/mean_terminated_length": 740.2440795898438, + "completions/min_length": 155.0, + "completions/min_terminated_length": 155.0, + "epoch": 0.8261500384057353, + "grad_norm": 2.1252243518829346, + "kl": 5.3671875, + "learning_rate": 1.805379379214093e-07, + "loss": 0.3033, + "num_tokens": 1300113073.0, + "reward": 1.9208984375, + "reward_std": 0.4738973379135132, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.8984375, + "rewards/format_reward/std": 0.30236753821372986, + "rewards/tag_count_reward/mean": 0.9462890625, + "rewards/tag_count_reward/std": 0.16342678666114807, + "step": 2420 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2024.0, + "completions/mean_length": 872.5234375, + "completions/mean_terminated_length": 814.7130737304688, + "completions/min_length": 12.0, + "completions/min_terminated_length": 12.0, + "epoch": 0.826491422719126, + "grad_norm": 3.4147696495056152, + "kl": 7.90625, + "learning_rate": 1.802321417022899e-07, + "loss": 0.4664, + "num_tokens": 1300635661.0, + "reward": 1.79150390625, + "reward_std": 0.5270118117332458, + "rewards/accuracy_reward/mean": 0.029296875, + "rewards/accuracy_reward/std": 0.16880230605602264, + "rewards/format_reward/mean": 0.841796875, + "rewards/format_reward/std": 0.36528825759887695, + "rewards/tag_count_reward/mean": 0.92041015625, + "rewards/tag_count_reward/std": 0.19631743431091309, + "step": 2421 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1981.0, + "completions/mean_length": 908.337890625, + "completions/mean_terminated_length": 869.197998046875, + "completions/min_length": 177.0, + "completions/min_terminated_length": 177.0, + "epoch": 0.8268328070325168, + "grad_norm": 0.9701458215713501, + "kl": 6.109375, + "learning_rate": 1.7992687030101388e-07, + "loss": 0.3619, + "num_tokens": 1301183578.0, + "reward": 1.8291015625, + "reward_std": 0.5332648754119873, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.859375, + "rewards/format_reward/std": 0.3479743003845215, + "rewards/tag_count_reward/mean": 0.9228515625, + "rewards/tag_count_reward/std": 0.1988353431224823, + "step": 2422 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1910.0, + "completions/mean_length": 765.150390625, + "completions/mean_terminated_length": 749.9387817382812, + "completions/min_length": 114.0, + "completions/min_terminated_length": 114.0, + "epoch": 0.8271741913459076, + "grad_norm": 1.2732343673706055, + "kl": 4.94140625, + "learning_rate": 1.7962212415085804e-07, + "loss": 0.327, + "num_tokens": 1301652647.0, + "reward": 1.9091796875, + "reward_std": 0.4718356728553772, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.8984375, + "rewards/format_reward/std": 0.30236753821372986, + "rewards/tag_count_reward/mean": 0.9423828125, + "rewards/tag_count_reward/std": 0.17374257743358612, + "step": 2423 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1978.0, + "completions/mean_length": 805.751953125, + "completions/mean_terminated_length": 775.9380493164062, + "completions/min_length": 155.0, + "completions/min_terminated_length": 155.0, + "epoch": 0.8275155756592985, + "grad_norm": 1.3992562294006348, + "kl": 4.984375, + "learning_rate": 1.7931790368435403e-07, + "loss": 0.3079, + "num_tokens": 1302144072.0, + "reward": 1.89111328125, + "reward_std": 0.5150551795959473, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.94189453125, + "rewards/tag_count_reward/std": 0.17533229291439056, + "step": 2424 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1947.0, + "completions/mean_length": 795.474609375, + "completions/mean_terminated_length": 778.1129150390625, + "completions/min_length": 5.0, + "completions/min_terminated_length": 5.0, + "epoch": 0.8278569599726893, + "grad_norm": 0.9801636934280396, + "kl": 4.6953125, + "learning_rate": 1.7901420933328696e-07, + "loss": 0.2657, + "num_tokens": 1302628315.0, + "reward": 1.9033203125, + "reward_std": 0.46424224972724915, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.873046875, + "rewards/format_reward/std": 0.33324605226516724, + "rewards/tag_count_reward/mean": 0.9462890625, + "rewards/tag_count_reward/std": 0.15964092314243317, + "step": 2425 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1986.0, + "completions/mean_length": 776.330078125, + "completions/mean_terminated_length": 756.1448974609375, + "completions/min_length": 154.0, + "completions/min_terminated_length": 154.0, + "epoch": 0.8281983442860801, + "grad_norm": 2.3696813583374023, + "kl": 4.9140625, + "learning_rate": 1.787110415286956e-07, + "loss": 0.3149, + "num_tokens": 1303104116.0, + "reward": 1.95068359375, + "reward_std": 0.5256731510162354, + "rewards/accuracy_reward/mean": 0.123046875, + "rewards/accuracy_reward/std": 0.32881227135658264, + "rewards/format_reward/mean": 0.88671875, + "rewards/format_reward/std": 0.3172462284564972, + "rewards/tag_count_reward/mean": 0.94091796875, + "rewards/tag_count_reward/std": 0.17570249736309052, + "step": 2426 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1974.0, + "completions/mean_length": 863.568359375, + "completions/mean_terminated_length": 812.910400390625, + "completions/min_length": 161.0, + "completions/min_terminated_length": 161.0, + "epoch": 0.8285397285994709, + "grad_norm": 1.6582800149917603, + "kl": 5.95703125, + "learning_rate": 1.784084007008711e-07, + "loss": 0.406, + "num_tokens": 1303623975.0, + "reward": 1.90966796875, + "reward_std": 0.5016525983810425, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.87890625, + "rewards/format_reward/std": 0.3265552520751953, + "rewards/tag_count_reward/mean": 0.94091796875, + "rewards/tag_count_reward/std": 0.16859769821166992, + "step": 2427 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 834.3359375, + "completions/mean_terminated_length": 777.2515258789062, + "completions/min_length": 48.0, + "completions/min_terminated_length": 48.0, + "epoch": 0.8288811129128617, + "grad_norm": 0.9909442663192749, + "kl": 6.7734375, + "learning_rate": 1.781062872793567e-07, + "loss": 0.4367, + "num_tokens": 1304125779.0, + "reward": 1.86865234375, + "reward_std": 0.5444897413253784, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.92919921875, + "rewards/tag_count_reward/std": 0.19343921542167664, + "step": 2428 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1979.0, + "completions/mean_length": 765.71484375, + "completions/mean_terminated_length": 750.5098876953125, + "completions/min_length": 151.0, + "completions/min_terminated_length": 151.0, + "epoch": 0.8292224972262524, + "grad_norm": 1.1289061307907104, + "kl": 4.28515625, + "learning_rate": 1.778047016929473e-07, + "loss": 0.2583, + "num_tokens": 1304594449.0, + "reward": 1.986328125, + "reward_std": 0.4804825782775879, + "rewards/accuracy_reward/mean": 0.12890625, + "rewards/accuracy_reward/std": 0.33542385697364807, + "rewards/format_reward/mean": 0.904296875, + "rewards/format_reward/std": 0.2944713830947876, + "rewards/tag_count_reward/mean": 0.953125, + "rewards/tag_count_reward/std": 0.15483088791370392, + "step": 2429 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 811.1171875, + "completions/mean_terminated_length": 791.4841918945312, + "completions/min_length": 168.0, + "completions/min_terminated_length": 168.0, + "epoch": 0.8295638815396432, + "grad_norm": 1.3904825448989868, + "kl": 4.9140625, + "learning_rate": 1.7750364436968836e-07, + "loss": 0.3166, + "num_tokens": 1305084429.0, + "reward": 1.89697265625, + "reward_std": 0.521807074546814, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.93994140625, + "rewards/tag_count_reward/std": 0.16897699236869812, + "step": 2430 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 791.181640625, + "completions/mean_terminated_length": 758.4389038085938, + "completions/min_length": 108.0, + "completions/min_terminated_length": 108.0, + "epoch": 0.829905265853034, + "grad_norm": 1.7678790092468262, + "kl": 6.421875, + "learning_rate": 1.7720311573687575e-07, + "loss": 0.4022, + "num_tokens": 1305567786.0, + "reward": 1.92236328125, + "reward_std": 0.5290135145187378, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.884765625, + "rewards/format_reward/std": 0.3196168541908264, + "rewards/tag_count_reward/mean": 0.94189453125, + "rewards/tag_count_reward/std": 0.1739315241575241, + "step": 2431 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1937.0, + "completions/mean_length": 828.103515625, + "completions/mean_terminated_length": 788.7520141601562, + "completions/min_length": 209.0, + "completions/min_terminated_length": 209.0, + "epoch": 0.8302466501664248, + "grad_norm": 1.0657150745391846, + "kl": 6.5625, + "learning_rate": 1.769031162210548e-07, + "loss": 0.4132, + "num_tokens": 1306068479.0, + "reward": 1.88134765625, + "reward_std": 0.4663490653038025, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.8828125, + "rewards/format_reward/std": 0.32195815443992615, + "rewards/tag_count_reward/mean": 0.94189453125, + "rewards/tag_count_reward/std": 0.16893739998340607, + "step": 2432 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 847.29296875, + "completions/mean_terminated_length": 793.3836669921875, + "completions/min_length": 71.0, + "completions/min_terminated_length": 71.0, + "epoch": 0.8305880344798157, + "grad_norm": 3.5841877460479736, + "kl": 8.984375, + "learning_rate": 1.766036462480201e-07, + "loss": 0.5267, + "num_tokens": 1306582661.0, + "reward": 1.79296875, + "reward_std": 0.5832004547119141, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.81640625, + "rewards/format_reward/std": 0.3875311613082886, + "rewards/tag_count_reward/mean": 0.91015625, + "rewards/tag_count_reward/std": 0.21154166758060455, + "step": 2433 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2004.0, + "completions/mean_length": 838.97265625, + "completions/mean_terminated_length": 809.9560546875, + "completions/min_length": 115.0, + "completions/min_terminated_length": 115.0, + "epoch": 0.8309294187932065, + "grad_norm": 2.315011978149414, + "kl": 6.375, + "learning_rate": 1.7630470624281442e-07, + "loss": 0.3725, + "num_tokens": 1307092823.0, + "reward": 1.84716796875, + "reward_std": 0.5111981630325317, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.8671875, + "rewards/format_reward/std": 0.33970388770103455, + "rewards/tag_count_reward/mean": 0.93701171875, + "rewards/tag_count_reward/std": 0.1791812628507614, + "step": 2434 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1958.0, + "completions/mean_length": 872.875, + "completions/mean_terminated_length": 830.0567016601562, + "completions/min_length": 223.0, + "completions/min_terminated_length": 223.0, + "epoch": 0.8312708031065973, + "grad_norm": 2.2868504524230957, + "kl": 7.734375, + "learning_rate": 1.7600629662972832e-07, + "loss": 0.4684, + "num_tokens": 1307613991.0, + "reward": 1.8720703125, + "reward_std": 0.5721770524978638, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.9267578125, + "rewards/tag_count_reward/std": 0.19785255193710327, + "step": 2435 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2001.0, + "completions/mean_length": 820.611328125, + "completions/mean_terminated_length": 778.4586181640625, + "completions/min_length": 135.0, + "completions/min_terminated_length": 135.0, + "epoch": 0.8316121874199881, + "grad_norm": 2.0491178035736084, + "kl": 5.859375, + "learning_rate": 1.757084178322999e-07, + "loss": 0.3588, + "num_tokens": 1308115648.0, + "reward": 1.861328125, + "reward_std": 0.48040178418159485, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.875, + "rewards/format_reward/std": 0.3310423493385315, + "rewards/tag_count_reward/mean": 0.947265625, + "rewards/tag_count_reward/std": 0.16224436461925507, + "step": 2436 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2004.0, + "completions/mean_length": 819.5078125, + "completions/mean_terminated_length": 784.9718627929688, + "completions/min_length": 130.0, + "completions/min_terminated_length": 130.0, + "epoch": 0.8319535717333788, + "grad_norm": 1.4329066276550293, + "kl": 6.2109375, + "learning_rate": 1.754110702733134e-07, + "loss": 0.3724, + "num_tokens": 1308608388.0, + "reward": 1.87744140625, + "reward_std": 0.4910423159599304, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.87890625, + "rewards/format_reward/std": 0.3265552520751953, + "rewards/tag_count_reward/mean": 0.93603515625, + "rewards/tag_count_reward/std": 0.1781490594148636, + "step": 2437 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1892.0, + "completions/mean_length": 813.140625, + "completions/mean_terminated_length": 770.7313232421875, + "completions/min_length": 123.0, + "completions/min_terminated_length": 123.0, + "epoch": 0.8322949560467696, + "grad_norm": 0.8658393025398254, + "kl": 5.140625, + "learning_rate": 1.7511425437479946e-07, + "loss": 0.3328, + "num_tokens": 1309106924.0, + "reward": 1.90234375, + "reward_std": 0.44942134618759155, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.88671875, + "rewards/format_reward/std": 0.3172462284564972, + "rewards/tag_count_reward/mean": 0.9453125, + "rewards/tag_count_reward/std": 0.1623503416776657, + "step": 2438 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1975.0, + "completions/mean_length": 892.69140625, + "completions/mean_terminated_length": 843.279052734375, + "completions/min_length": 147.0, + "completions/min_terminated_length": 147.0, + "epoch": 0.8326363403601604, + "grad_norm": 1.0234296321868896, + "kl": 6.0234375, + "learning_rate": 1.7481797055803382e-07, + "loss": 0.3668, + "num_tokens": 1309645566.0, + "reward": 1.87255859375, + "reward_std": 0.49502408504486084, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.8671875, + "rewards/format_reward/std": 0.33970388770103455, + "rewards/tag_count_reward/mean": 0.93701171875, + "rewards/tag_count_reward/std": 0.18456129729747772, + "step": 2439 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2024.0, + "completions/mean_length": 818.84375, + "completions/mean_terminated_length": 781.7464599609375, + "completions/min_length": 157.0, + "completions/min_terminated_length": 157.0, + "epoch": 0.8329777246735512, + "grad_norm": 2.0106232166290283, + "kl": 5.4921875, + "learning_rate": 1.7452221924353733e-07, + "loss": 0.3998, + "num_tokens": 1310140414.0, + "reward": 1.87109375, + "reward_std": 0.45293596386909485, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.884765625, + "rewards/format_reward/std": 0.3196168541908264, + "rewards/tag_count_reward/mean": 0.943359375, + "rewards/tag_count_reward/std": 0.1726529598236084, + "step": 2440 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1815.0, + "completions/mean_length": 756.9453125, + "completions/mean_terminated_length": 723.3106079101562, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "epoch": 0.8333191089869421, + "grad_norm": 2.217083692550659, + "kl": 4.92578125, + "learning_rate": 1.7422700085107485e-07, + "loss": 0.3484, + "num_tokens": 1310609538.0, + "reward": 1.97412109375, + "reward_std": 0.47586676478385925, + "rewards/accuracy_reward/mean": 0.10685484111309052, + "rewards/accuracy_reward/std": 0.3092404901981354, + "rewards/format_reward/mean": 0.9140625, + "rewards/format_reward/std": 0.28054583072662354, + "rewards/tag_count_reward/mean": 0.95654296875, + "rewards/tag_count_reward/std": 0.16008546948432922, + "step": 2441 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2030.0, + "completions/mean_length": 802.890625, + "completions/mean_terminated_length": 783.1270141601562, + "completions/min_length": 133.0, + "completions/min_terminated_length": 133.0, + "epoch": 0.8336604933003329, + "grad_norm": 1.6382285356521606, + "kl": 4.71875, + "learning_rate": 1.7393231579965467e-07, + "loss": 0.2743, + "num_tokens": 1311097066.0, + "reward": 1.8720703125, + "reward_std": 0.4506915807723999, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.86328125, + "rewards/format_reward/std": 0.3438861668109894, + "rewards/tag_count_reward/mean": 0.9423828125, + "rewards/tag_count_reward/std": 0.16874286532402039, + "step": 2442 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2019.0, + "completions/mean_length": 884.443359375, + "completions/mean_terminated_length": 832.2020263671875, + "completions/min_length": 44.0, + "completions/min_terminated_length": 44.0, + "epoch": 0.8340018776137237, + "grad_norm": 1.4400036334991455, + "kl": 6.08203125, + "learning_rate": 1.736381645075286e-07, + "loss": 0.3848, + "num_tokens": 1311640045.0, + "reward": 1.79443359375, + "reward_std": 0.514877200126648, + "rewards/accuracy_reward/mean": 0.0234375, + "rewards/accuracy_reward/std": 0.15143637359142303, + "rewards/format_reward/mean": 0.849609375, + "rewards/format_reward/std": 0.35780346393585205, + "rewards/tag_count_reward/mean": 0.92138671875, + "rewards/tag_count_reward/std": 0.20283371210098267, + "step": 2443 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 858.203125, + "completions/mean_terminated_length": 822.2937622070312, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "epoch": 0.8343432619271145, + "grad_norm": 1.732412576675415, + "kl": 6.8828125, + "learning_rate": 1.733445473921904e-07, + "loss": 0.4102, + "num_tokens": 1312155797.0, + "reward": 1.83447265625, + "reward_std": 0.5208526849746704, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.857421875, + "rewards/format_reward/std": 0.3499840497970581, + "rewards/tag_count_reward/mean": 0.93212890625, + "rewards/tag_count_reward/std": 0.18415184319019318, + "step": 2444 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2017.0, + "completions/mean_length": 802.6953125, + "completions/mean_terminated_length": 752.0731201171875, + "completions/min_length": 135.0, + "completions/min_terminated_length": 135.0, + "epoch": 0.8346846462405052, + "grad_norm": 0.8828231692314148, + "kl": 7.34375, + "learning_rate": 1.7305146487037603e-07, + "loss": 0.4933, + "num_tokens": 1312640825.0, + "reward": 1.91259765625, + "reward_std": 0.544389545917511, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.875, + "rewards/format_reward/std": 0.3310423493385315, + "rewards/tag_count_reward/mean": 0.93212890625, + "rewards/tag_count_reward/std": 0.19574224948883057, + "step": 2445 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1999.0, + "completions/mean_length": 851.515625, + "completions/mean_terminated_length": 802.8779907226562, + "completions/min_length": 149.0, + "completions/min_terminated_length": 149.0, + "epoch": 0.835026030553896, + "grad_norm": 2.0713346004486084, + "kl": 8.328125, + "learning_rate": 1.727589173580625e-07, + "loss": 0.5235, + "num_tokens": 1313149937.0, + "reward": 1.80859375, + "reward_std": 0.5601276159286499, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.83203125, + "rewards/format_reward/std": 0.374204158782959, + "rewards/tag_count_reward/mean": 0.919921875, + "rewards/tag_count_reward/std": 0.19890500605106354, + "step": 2446 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 861.833984375, + "completions/mean_terminated_length": 838.2052001953125, + "completions/min_length": 122.0, + "completions/min_terminated_length": 122.0, + "epoch": 0.8353674148672868, + "grad_norm": 0.9768324494361877, + "kl": 5.34765625, + "learning_rate": 1.724669052704673e-07, + "loss": 0.3107, + "num_tokens": 1313666588.0, + "reward": 1.88671875, + "reward_std": 0.5045324563980103, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.873046875, + "rewards/format_reward/std": 0.33324605226516724, + "rewards/tag_count_reward/mean": 0.939453125, + "rewards/tag_count_reward/std": 0.17203198373317719, + "step": 2447 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2020.0, + "completions/mean_length": 779.828125, + "completions/mean_terminated_length": 738.9193115234375, + "completions/min_length": 154.0, + "completions/min_terminated_length": 154.0, + "epoch": 0.8357087991806776, + "grad_norm": 1.2612099647521973, + "kl": 6.4609375, + "learning_rate": 1.7217542902204847e-07, + "loss": 0.4308, + "num_tokens": 1314136148.0, + "reward": 1.9033203125, + "reward_std": 0.4736611843109131, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.89453125, + "rewards/format_reward/std": 0.3074568510055542, + "rewards/tag_count_reward/mean": 0.9443359375, + "rewards/tag_count_reward/std": 0.1764710247516632, + "step": 2448 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1961.0, + "completions/mean_length": 815.84375, + "completions/mean_terminated_length": 760.5223999023438, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "epoch": 0.8360501834940685, + "grad_norm": 0.8281819820404053, + "kl": 6.25, + "learning_rate": 1.7188448902650287e-07, + "loss": 0.3747, + "num_tokens": 1314636884.0, + "reward": 1.8671875, + "reward_std": 0.5257279276847839, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.93359375, + "rewards/tag_count_reward/std": 0.190873920917511, + "step": 2449 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 798.275390625, + "completions/mean_terminated_length": 765.7174682617188, + "completions/min_length": 107.0, + "completions/min_terminated_length": 107.0, + "epoch": 0.8363915678074593, + "grad_norm": 0.9520702958106995, + "kl": 5.30078125, + "learning_rate": 1.7159408569676704e-07, + "loss": 0.3225, + "num_tokens": 1315122033.0, + "reward": 1.873046875, + "reward_std": 0.49339762330055237, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.875, + "rewards/format_reward/std": 0.3310423493385315, + "rewards/tag_count_reward/mean": 0.9375, + "rewards/tag_count_reward/std": 0.18373169004917145, + "step": 2450 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1920.0, + "completions/mean_length": 820.03515625, + "completions/mean_terminated_length": 764.9019775390625, + "completions/min_length": 134.0, + "completions/min_terminated_length": 134.0, + "epoch": 0.8367329521208501, + "grad_norm": 1.0722157955169678, + "kl": 6.51171875, + "learning_rate": 1.71304219445015e-07, + "loss": 0.44, + "num_tokens": 1315613059.0, + "reward": 1.89208984375, + "reward_std": 0.5305795669555664, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.93701171875, + "rewards/tag_count_reward/std": 0.18189117312431335, + "step": 2451 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1933.0, + "completions/mean_length": 834.470703125, + "completions/mean_terminated_length": 797.8450317382812, + "completions/min_length": 138.0, + "completions/min_terminated_length": 138.0, + "epoch": 0.8370743364342409, + "grad_norm": 1.146774172782898, + "kl": 5.04296875, + "learning_rate": 1.7101489068265935e-07, + "loss": 0.3122, + "num_tokens": 1316124084.0, + "reward": 1.91845703125, + "reward_std": 0.46381455659866333, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.8984375, + "rewards/format_reward/std": 0.30236753821372986, + "rewards/tag_count_reward/mean": 0.94970703125, + "rewards/tag_count_reward/std": 0.16339389979839325, + "step": 2452 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 791.599609375, + "completions/mean_terminated_length": 756.2791137695312, + "completions/min_length": 164.0, + "completions/min_terminated_length": 164.0, + "epoch": 0.8374157207476316, + "grad_norm": 2.683335781097412, + "kl": 5.53515625, + "learning_rate": 1.7072609982034874e-07, + "loss": 0.3822, + "num_tokens": 1316606215.0, + "reward": 1.84228515625, + "reward_std": 0.4803961515426636, + "rewards/accuracy_reward/mean": 0.03125, + "rewards/accuracy_reward/std": 0.17416280508041382, + "rewards/format_reward/mean": 0.87890625, + "rewards/format_reward/std": 0.3265552520751953, + "rewards/tag_count_reward/mean": 0.93212890625, + "rewards/tag_count_reward/std": 0.18939071893692017, + "step": 2453 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1946.0, + "completions/mean_length": 807.419921875, + "completions/mean_terminated_length": 780.181640625, + "completions/min_length": 141.0, + "completions/min_terminated_length": 141.0, + "epoch": 0.8377571050610224, + "grad_norm": 0.8453850150108337, + "kl": 5.09375, + "learning_rate": 1.7043784726796934e-07, + "loss": 0.3165, + "num_tokens": 1317093790.0, + "reward": 1.89599609375, + "reward_std": 0.43774914741516113, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.88671875, + "rewards/format_reward/std": 0.3172462284564972, + "rewards/tag_count_reward/mean": 0.94287109375, + "rewards/tag_count_reward/std": 0.17070980370044708, + "step": 2454 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1880.0, + "completions/mean_length": 817.33984375, + "completions/mean_terminated_length": 769.9107055664062, + "completions/min_length": 124.0, + "completions/min_terminated_length": 124.0, + "epoch": 0.8380984893744132, + "grad_norm": 0.8198954463005066, + "kl": 6.859375, + "learning_rate": 1.7015013343464302e-07, + "loss": 0.4087, + "num_tokens": 1317593932.0, + "reward": 1.8173828125, + "reward_std": 0.5331565141677856, + "rewards/accuracy_reward/mean": 0.03515625, + "rewards/accuracy_reward/std": 0.1843547374010086, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.9267578125, + "rewards/tag_count_reward/std": 0.1941080093383789, + "step": 2455 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2024.0, + "completions/mean_length": 722.31640625, + "completions/mean_terminated_length": 698.5963745117188, + "completions/min_length": 160.0, + "completions/min_terminated_length": 160.0, + "epoch": 0.838439873687804, + "grad_norm": 1.1952383518218994, + "kl": 4.9921875, + "learning_rate": 1.698629587287266e-07, + "loss": 0.2944, + "num_tokens": 1318038894.0, + "reward": 1.93896484375, + "reward_std": 0.4384310245513916, + "rewards/accuracy_reward/mean": 0.0786290317773819, + "rewards/accuracy_reward/std": 0.26943066716194153, + "rewards/format_reward/mean": 0.912109375, + "rewards/format_reward/std": 0.2834126651287079, + "rewards/tag_count_reward/mean": 0.95068359375, + "rewards/tag_count_reward/std": 0.15991228818893433, + "step": 2456 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2022.0, + "completions/mean_length": 754.697265625, + "completions/mean_terminated_length": 726.3013916015625, + "completions/min_length": 8.0, + "completions/min_terminated_length": 8.0, + "epoch": 0.8387812580011949, + "grad_norm": 1.8518648147583008, + "kl": 5.53125, + "learning_rate": 1.6957632355781243e-07, + "loss": 0.2906, + "num_tokens": 1318494899.0, + "reward": 1.955078125, + "reward_std": 0.4595924913883209, + "rewards/accuracy_reward/mean": 0.134765625, + "rewards/accuracy_reward/std": 0.3418070077896118, + "rewards/format_reward/mean": 0.87890625, + "rewards/format_reward/std": 0.3265552520751953, + "rewards/tag_count_reward/mean": 0.94140625, + "rewards/tag_count_reward/std": 0.17411890625953674, + "step": 2457 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1908.0, + "completions/mean_length": 794.3828125, + "completions/mean_terminated_length": 761.7234497070312, + "completions/min_length": 150.0, + "completions/min_terminated_length": 150.0, + "epoch": 0.8391226423145857, + "grad_norm": 1.0499241352081299, + "kl": 5.875, + "learning_rate": 1.6929022832872653e-07, + "loss": 0.3696, + "num_tokens": 1318979223.0, + "reward": 1.9111328125, + "reward_std": 0.4973984956741333, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.8828125, + "rewards/format_reward/std": 0.32195815443992615, + "rewards/tag_count_reward/mean": 0.9423828125, + "rewards/tag_count_reward/std": 0.17514485120773315, + "step": 2458 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1803.0, + "completions/mean_length": 747.6796875, + "completions/mean_terminated_length": 724.4135131835938, + "completions/min_length": 13.0, + "completions/min_terminated_length": 13.0, + "epoch": 0.8394640266279765, + "grad_norm": 0.9654370546340942, + "kl": 5.171875, + "learning_rate": 1.6900467344752872e-07, + "loss": 0.2651, + "num_tokens": 1319440211.0, + "reward": 1.9091796875, + "reward_std": 0.45924073457717896, + "rewards/accuracy_reward/mean": 0.07258064299821854, + "rewards/accuracy_reward/std": 0.25970885157585144, + "rewards/format_reward/mean": 0.88671875, + "rewards/format_reward/std": 0.3172462284564972, + "rewards/tag_count_reward/mean": 0.9521484375, + "rewards/tag_count_reward/std": 0.15133222937583923, + "step": 2459 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1948.0, + "completions/mean_length": 790.044921875, + "completions/mean_terminated_length": 757.2725830078125, + "completions/min_length": 119.0, + "completions/min_terminated_length": 119.0, + "epoch": 0.8398054109413673, + "grad_norm": 0.9977067708969116, + "kl": 6.08984375, + "learning_rate": 1.6871965931951178e-07, + "loss": 0.3605, + "num_tokens": 1319916170.0, + "reward": 1.85400390625, + "reward_std": 0.4964134097099304, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.93408203125, + "rewards/tag_count_reward/std": 0.19136326014995575, + "step": 2460 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1907.0, + "completions/mean_length": 755.177734375, + "completions/mean_terminated_length": 716.158935546875, + "completions/min_length": 159.0, + "completions/min_terminated_length": 159.0, + "epoch": 0.840146795254758, + "grad_norm": 1.0653966665267944, + "kl": 7.1015625, + "learning_rate": 1.684351863492014e-07, + "loss": 0.4232, + "num_tokens": 1320371173.0, + "reward": 1.837890625, + "reward_std": 0.6131343841552734, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.8359375, + "rewards/format_reward/std": 0.37069445848464966, + "rewards/tag_count_reward/mean": 0.91796875, + "rewards/tag_count_reward/std": 0.2011692225933075, + "step": 2461 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 812.6015625, + "completions/mean_terminated_length": 777.8714599609375, + "completions/min_length": 152.0, + "completions/min_terminated_length": 152.0, + "epoch": 0.8404881795681488, + "grad_norm": 0.7508025169372559, + "kl": 5.80859375, + "learning_rate": 1.6815125494035494e-07, + "loss": 0.379, + "num_tokens": 1320861273.0, + "reward": 1.857421875, + "reward_std": 0.42223262786865234, + "rewards/accuracy_reward/mean": 0.02822580561041832, + "rewards/accuracy_reward/std": 0.1657845675945282, + "rewards/format_reward/mean": 0.888671875, + "rewards/format_reward/std": 0.31484565138816833, + "rewards/tag_count_reward/mean": 0.94140625, + "rewards/tag_count_reward/std": 0.18302303552627563, + "step": 2462 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1952.0, + "completions/mean_length": 814.798828125, + "completions/mean_terminated_length": 782.6713256835938, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "epoch": 0.8408295638815396, + "grad_norm": 1.774247169494629, + "kl": 4.22265625, + "learning_rate": 1.678678654959609e-07, + "loss": 0.2615, + "num_tokens": 1321359538.0, + "reward": 1.869140625, + "reward_std": 0.48968732357025146, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.880859375, + "rewards/format_reward/std": 0.32427072525024414, + "rewards/tag_count_reward/mean": 0.9375, + "rewards/tag_count_reward/std": 0.18373169004917145, + "step": 2463 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2001.0, + "completions/mean_length": 772.46484375, + "completions/mean_terminated_length": 736.6063842773438, + "completions/min_length": 150.0, + "completions/min_terminated_length": 150.0, + "epoch": 0.8411709481949304, + "grad_norm": 3.33648943901062, + "kl": 5.1328125, + "learning_rate": 1.6758501841823902e-07, + "loss": 0.3516, + "num_tokens": 1321830304.0, + "reward": 1.8837890625, + "reward_std": 0.48854079842567444, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.884765625, + "rewards/format_reward/std": 0.3196168541908264, + "rewards/tag_count_reward/mean": 0.9384765625, + "rewards/tag_count_reward/std": 0.17866657674312592, + "step": 2464 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 838.130859375, + "completions/mean_terminated_length": 788.9491577148438, + "completions/min_length": 164.0, + "completions/min_terminated_length": 164.0, + "epoch": 0.8415123325083212, + "grad_norm": 0.8046271204948425, + "kl": 5.984375, + "learning_rate": 1.6730271410863864e-07, + "loss": 0.3562, + "num_tokens": 1322348707.0, + "reward": 1.83251953125, + "reward_std": 0.5146738290786743, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.8671875, + "rewards/format_reward/std": 0.33970388770103455, + "rewards/tag_count_reward/mean": 0.92626953125, + "rewards/tag_count_reward/std": 0.19859671592712402, + "step": 2465 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1993.0, + "completions/mean_length": 772.365234375, + "completions/mean_terminated_length": 739.1322631835938, + "completions/min_length": 165.0, + "completions/min_terminated_length": 165.0, + "epoch": 0.8418537168217121, + "grad_norm": 1.6727104187011719, + "kl": 3.3203125, + "learning_rate": 1.6702095296783942e-07, + "loss": 0.2213, + "num_tokens": 1322823614.0, + "reward": 1.951171875, + "reward_std": 0.41594749689102173, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.916015625, + "rewards/format_reward/std": 0.2776356339454651, + "rewards/tag_count_reward/mean": 0.958984375, + "rewards/tag_count_reward/std": 0.14764074981212616, + "step": 2466 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2019.0, + "completions/mean_length": 803.171875, + "completions/mean_terminated_length": 778.37451171875, + "completions/min_length": 72.0, + "completions/min_terminated_length": 72.0, + "epoch": 0.8421951011351029, + "grad_norm": 1.9509830474853516, + "kl": 3.86328125, + "learning_rate": 1.6673973539574953e-07, + "loss": 0.2425, + "num_tokens": 1323309702.0, + "reward": 1.9228515625, + "reward_std": 0.433314710855484, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.90234375, + "rewards/format_reward/std": 0.29713961482048035, + "rewards/tag_count_reward/mean": 0.9521484375, + "rewards/tag_count_reward/std": 0.1576654314994812, + "step": 2467 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1876.0, + "completions/mean_length": 851.47265625, + "completions/mean_terminated_length": 807.87451171875, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "epoch": 0.8425364854484937, + "grad_norm": 1.1834297180175781, + "kl": 6.5078125, + "learning_rate": 1.6645906179150592e-07, + "loss": 0.3857, + "num_tokens": 1323829128.0, + "reward": 1.8564453125, + "reward_std": 0.5604408979415894, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.9248046875, + "rewards/tag_count_reward/std": 0.19524674117565155, + "step": 2468 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2028.0, + "completions/mean_length": 800.455078125, + "completions/mean_terminated_length": 762.8027954101562, + "completions/min_length": 110.0, + "completions/min_terminated_length": 110.0, + "epoch": 0.8428778697618844, + "grad_norm": 1.1407212018966675, + "kl": 6.34765625, + "learning_rate": 1.661789325534737e-07, + "loss": 0.4454, + "num_tokens": 1324316465.0, + "reward": 1.904296875, + "reward_std": 0.48995035886764526, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.89453125, + "rewards/format_reward/std": 0.3074568510055542, + "rewards/tag_count_reward/mean": 0.939453125, + "rewards/tag_count_reward/std": 0.18171313405036926, + "step": 2469 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1890.0, + "completions/mean_length": 773.349609375, + "completions/mean_terminated_length": 707.9158325195312, + "completions/min_length": 118.0, + "completions/min_terminated_length": 118.0, + "epoch": 0.8432192540752752, + "grad_norm": 1.5480372905731201, + "kl": 7.921875, + "learning_rate": 1.6589934807924482e-07, + "loss": 0.5114, + "num_tokens": 1324782356.0, + "reward": 1.90087890625, + "reward_std": 0.6045435667037964, + "rewards/accuracy_reward/mean": 0.12298387289047241, + "rewards/accuracy_reward/std": 0.32875028252601624, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.92626953125, + "rewards/tag_count_reward/std": 0.1979798823595047, + "step": 2470 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1991.0, + "completions/mean_length": 728.19921875, + "completions/mean_terminated_length": 707.2500610351562, + "completions/min_length": 108.0, + "completions/min_terminated_length": 108.0, + "epoch": 0.843560638388666, + "grad_norm": 1.6724046468734741, + "kl": 6.765625, + "learning_rate": 1.6562030876563843e-07, + "loss": 0.375, + "num_tokens": 1325229978.0, + "reward": 1.828125, + "reward_std": 0.5169737339019775, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.927734375, + "rewards/tag_count_reward/std": 0.18546061217784882, + "step": 2471 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1889.0, + "completions/mean_length": 785.814453125, + "completions/mean_terminated_length": 742.4666748046875, + "completions/min_length": 83.0, + "completions/min_terminated_length": 83.0, + "epoch": 0.8439020227020568, + "grad_norm": 2.92116641998291, + "kl": 7.4765625, + "learning_rate": 1.653418150086996e-07, + "loss": 0.4605, + "num_tokens": 1325707051.0, + "reward": 1.8876953125, + "reward_std": 0.48363927006721497, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.873046875, + "rewards/format_reward/std": 0.33324605226516724, + "rewards/tag_count_reward/mean": 0.9365234375, + "rewards/tag_count_reward/std": 0.17934982478618622, + "step": 2472 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2017.0, + "completions/mean_length": 796.6875, + "completions/mean_terminated_length": 740.506103515625, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "epoch": 0.8442434070154476, + "grad_norm": 1.2109431028366089, + "kl": 7.7734375, + "learning_rate": 1.6506386720369953e-07, + "loss": 0.5195, + "num_tokens": 1326190651.0, + "reward": 1.89208984375, + "reward_std": 0.49778905510902405, + "rewards/accuracy_reward/mean": 0.10080645233392715, + "rewards/accuracy_reward/std": 0.30137622356414795, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.92919921875, + "rewards/tag_count_reward/std": 0.19343921542167664, + "step": 2473 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2031.0, + "completions/mean_length": 802.771484375, + "completions/mean_terminated_length": 744.2024536132812, + "completions/min_length": 147.0, + "completions/min_terminated_length": 147.0, + "epoch": 0.8445847913288385, + "grad_norm": 1.1114060878753662, + "kl": 7.3125, + "learning_rate": 1.6478646574513409e-07, + "loss": 0.4733, + "num_tokens": 1326674310.0, + "reward": 1.93017578125, + "reward_std": 0.6183890104293823, + "rewards/accuracy_reward/mean": 0.162109375, + "rewards/accuracy_reward/std": 0.3689115643501282, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.92431640625, + "rewards/tag_count_reward/std": 0.1941147744655609, + "step": 2474 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2021.0, + "completions/mean_length": 820.759765625, + "completions/mean_terminated_length": 778.6121826171875, + "completions/min_length": 152.0, + "completions/min_terminated_length": 152.0, + "epoch": 0.8449261756422293, + "grad_norm": 1.6558356285095215, + "kl": 5.71875, + "learning_rate": 1.6450961102672394e-07, + "loss": 0.3816, + "num_tokens": 1327182155.0, + "reward": 1.89111328125, + "reward_std": 0.4347790479660034, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.8984375, + "rewards/format_reward/std": 0.30236753821372986, + "rewards/tag_count_reward/mean": 0.94775390625, + "rewards/tag_count_reward/std": 0.16795603930950165, + "step": 2475 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2022.0, + "completions/mean_length": 789.876953125, + "completions/mean_terminated_length": 762.2534790039062, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "epoch": 0.8452675599556201, + "grad_norm": 1.3808722496032715, + "kl": 4.55078125, + "learning_rate": 1.6423330344141401e-07, + "loss": 0.2757, + "num_tokens": 1327672668.0, + "reward": 1.93115234375, + "reward_std": 0.5451623201370239, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.880859375, + "rewards/format_reward/std": 0.32427072525024414, + "rewards/tag_count_reward/mean": 0.94482421875, + "rewards/tag_count_reward/std": 0.1655435413122177, + "step": 2476 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 871.990234375, + "completions/mean_terminated_length": 819.1897583007812, + "completions/min_length": 173.0, + "completions/min_terminated_length": 173.0, + "epoch": 0.8456089442690108, + "grad_norm": 2.074889898300171, + "kl": 6.6328125, + "learning_rate": 1.6395754338137203e-07, + "loss": 0.4639, + "num_tokens": 1328196599.0, + "reward": 1.83984375, + "reward_std": 0.562311053276062, + "rewards/accuracy_reward/mean": 0.06854838877916336, + "rewards/accuracy_reward/std": 0.25293973088264465, + "rewards/format_reward/mean": 0.849609375, + "rewards/format_reward/std": 0.35780346393585205, + "rewards/tag_count_reward/mean": 0.923828125, + "rewards/tag_count_reward/std": 0.2010456770658493, + "step": 2477 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2007.0, + "completions/mean_length": 813.85546875, + "completions/mean_terminated_length": 768.8866577148438, + "completions/min_length": 169.0, + "completions/min_terminated_length": 169.0, + "epoch": 0.8459503285824016, + "grad_norm": 1.4364606142044067, + "kl": 6.109375, + "learning_rate": 1.6368233123798913e-07, + "loss": 0.4024, + "num_tokens": 1328692173.0, + "reward": 1.87109375, + "reward_std": 0.48487743735313416, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.875, + "rewards/format_reward/std": 0.3310423493385315, + "rewards/tag_count_reward/mean": 0.9375, + "rewards/tag_count_reward/std": 0.18571804463863373, + "step": 2478 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1981.0, + "completions/mean_length": 812.30859375, + "completions/mean_terminated_length": 775.0140380859375, + "completions/min_length": 127.0, + "completions/min_terminated_length": 127.0, + "epoch": 0.8462917128957924, + "grad_norm": 1.5719364881515503, + "kl": 6.1171875, + "learning_rate": 1.634076674018785e-07, + "loss": 0.3933, + "num_tokens": 1329174587.0, + "reward": 1.9228515625, + "reward_std": 0.5373904705047607, + "rewards/accuracy_reward/mean": 0.109375, + "rewards/accuracy_reward/std": 0.31241437792778015, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.9423828125, + "rewards/tag_count_reward/std": 0.17374257743358612, + "step": 2479 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2002.0, + "completions/mean_length": 839.46484375, + "completions/mean_terminated_length": 812.9301147460938, + "completions/min_length": 194.0, + "completions/min_terminated_length": 194.0, + "epoch": 0.8466330972091832, + "grad_norm": 0.9839436411857605, + "kl": 4.65234375, + "learning_rate": 1.6313355226287535e-07, + "loss": 0.2931, + "num_tokens": 1329679961.0, + "reward": 1.90185546875, + "reward_std": 0.4618559777736664, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.89453125, + "rewards/format_reward/std": 0.3074568510055542, + "rewards/tag_count_reward/mean": 0.94677734375, + "rewards/tag_count_reward/std": 0.16321250796318054, + "step": 2480 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2010.0, + "completions/mean_length": 814.62890625, + "completions/mean_terminated_length": 777.4044189453125, + "completions/min_length": 42.0, + "completions/min_terminated_length": 42.0, + "epoch": 0.846974481522574, + "grad_norm": 0.9385043382644653, + "kl": 4.55078125, + "learning_rate": 1.6285998621003581e-07, + "loss": 0.2869, + "num_tokens": 1330174619.0, + "reward": 1.89208984375, + "reward_std": 0.4626353085041046, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.89453125, + "rewards/format_reward/std": 0.3074568510055542, + "rewards/tag_count_reward/mean": 0.93896484375, + "rewards/tag_count_reward/std": 0.18718823790550232, + "step": 2481 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1863.0, + "completions/mean_length": 823.2890625, + "completions/mean_terminated_length": 776.0892333984375, + "completions/min_length": 171.0, + "completions/min_terminated_length": 171.0, + "epoch": 0.8473158658359649, + "grad_norm": 2.317014455795288, + "kl": 6.4921875, + "learning_rate": 1.6258696963163704e-07, + "loss": 0.4277, + "num_tokens": 1330675295.0, + "reward": 1.8251953125, + "reward_std": 0.52950119972229, + "rewards/accuracy_reward/mean": 0.02734375, + "rewards/accuracy_reward/std": 0.16324250400066376, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.9326171875, + "rewards/tag_count_reward/std": 0.19746580719947815, + "step": 2482 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1962.0, + "completions/mean_length": 872.4296875, + "completions/mean_terminated_length": 856.1347045898438, + "completions/min_length": 173.0, + "completions/min_terminated_length": 173.0, + "epoch": 0.8476572501493557, + "grad_norm": 1.13470458984375, + "kl": 4.11328125, + "learning_rate": 1.6231450291517617e-07, + "loss": 0.2317, + "num_tokens": 1331204923.0, + "reward": 1.9345703125, + "reward_std": 0.4377116560935974, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.900390625, + "rewards/format_reward/std": 0.29977133870124817, + "rewards/tag_count_reward/mean": 0.9521484375, + "rewards/tag_count_reward/std": 0.1576654314994812, + "step": 2483 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1879.0, + "completions/mean_length": 823.8828125, + "completions/mean_terminated_length": 776.7058715820312, + "completions/min_length": 88.0, + "completions/min_terminated_length": 88.0, + "epoch": 0.8479986344627465, + "grad_norm": 1.4106740951538086, + "kl": 6.3828125, + "learning_rate": 1.6204258644736966e-07, + "loss": 0.3612, + "num_tokens": 1331706143.0, + "reward": 1.89404296875, + "reward_std": 0.5347704887390137, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.93115234375, + "rewards/tag_count_reward/std": 0.19288021326065063, + "step": 2484 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2021.0, + "completions/mean_length": 794.443359375, + "completions/mean_terminated_length": 772.0138549804688, + "completions/min_length": 145.0, + "completions/min_terminated_length": 145.0, + "epoch": 0.8483400187761372, + "grad_norm": 1.2902165651321411, + "kl": 4.42578125, + "learning_rate": 1.6177122061415337e-07, + "loss": 0.2628, + "num_tokens": 1332190850.0, + "reward": 1.943359375, + "reward_std": 0.4758620262145996, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.8984375, + "rewards/format_reward/std": 0.30236753821372986, + "rewards/tag_count_reward/mean": 0.951171875, + "rewards/tag_count_reward/std": 0.15342977643013, + "step": 2485 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 803.541015625, + "completions/mean_terminated_length": 778.7510375976562, + "completions/min_length": 141.0, + "completions/min_terminated_length": 141.0, + "epoch": 0.848681403089528, + "grad_norm": 1.1515048742294312, + "kl": 4.984375, + "learning_rate": 1.615004058006812e-07, + "loss": 0.2792, + "num_tokens": 1332673479.0, + "reward": 1.91943359375, + "reward_std": 0.5097028017044067, + "rewards/accuracy_reward/mean": 0.09677419066429138, + "rewards/accuracy_reward/std": 0.2959485352039337, + "rewards/format_reward/mean": 0.8828125, + "rewards/format_reward/std": 0.32195815443992615, + "rewards/tag_count_reward/mean": 0.94287109375, + "rewards/tag_count_reward/std": 0.1749558448791504, + "step": 2486 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1709.0, + "completions/mean_length": 782.19140625, + "completions/mean_terminated_length": 749.2144165039062, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "epoch": 0.8490227874029188, + "grad_norm": 1.8417575359344482, + "kl": 6.6484375, + "learning_rate": 1.6123014239132568e-07, + "loss": 0.3942, + "num_tokens": 1333159577.0, + "reward": 1.93701171875, + "reward_std": 0.5437750816345215, + "rewards/accuracy_reward/mean": 0.115234375, + "rewards/accuracy_reward/std": 0.3196168541908264, + "rewards/format_reward/mean": 0.8828125, + "rewards/format_reward/std": 0.32195815443992615, + "rewards/tag_count_reward/mean": 0.93896484375, + "rewards/tag_count_reward/std": 0.17433211207389832, + "step": 2487 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1965.0, + "completions/mean_length": 876.380859375, + "completions/mean_terminated_length": 843.4437255859375, + "completions/min_length": 101.0, + "completions/min_terminated_length": 101.0, + "epoch": 0.8493641717163096, + "grad_norm": 2.1319403648376465, + "kl": 6.3046875, + "learning_rate": 1.6096043076967592e-07, + "loss": 0.3498, + "num_tokens": 1333679404.0, + "reward": 1.87890625, + "reward_std": 0.5440197587013245, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.93359375, + "rewards/tag_count_reward/std": 0.1876426339149475, + "step": 2488 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1989.0, + "completions/mean_length": 786.470703125, + "completions/mean_terminated_length": 761.3406372070312, + "completions/min_length": 147.0, + "completions/min_terminated_length": 147.0, + "epoch": 0.8497055560297004, + "grad_norm": 1.8529338836669922, + "kl": 6.3359375, + "learning_rate": 1.6069127131853846e-07, + "loss": 0.3665, + "num_tokens": 1334159837.0, + "reward": 1.91650390625, + "reward_std": 0.5342533588409424, + "rewards/accuracy_reward/mean": 0.111328125, + "rewards/accuracy_reward/std": 0.31484565138816833, + "rewards/format_reward/mean": 0.873046875, + "rewards/format_reward/std": 0.33324605226516724, + "rewards/tag_count_reward/mean": 0.93212890625, + "rewards/tag_count_reward/std": 0.18481481075286865, + "step": 2489 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1988.0, + "completions/mean_length": 891.326171875, + "completions/mean_terminated_length": 858.8092041015625, + "completions/min_length": 191.0, + "completions/min_terminated_length": 191.0, + "epoch": 0.8500469403430913, + "grad_norm": 1.5846630334854126, + "kl": 6.0703125, + "learning_rate": 1.6042266441993583e-07, + "loss": 0.3741, + "num_tokens": 1334687492.0, + "reward": 1.87841796875, + "reward_std": 0.49128904938697815, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.876953125, + "rewards/format_reward/std": 0.32881227135658264, + "rewards/tag_count_reward/mean": 0.93896484375, + "rewards/tag_count_reward/std": 0.17150279879570007, + "step": 2490 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1987.0, + "completions/mean_length": 762.4375, + "completions/mean_terminated_length": 731.5840454101562, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "epoch": 0.8503883246564821, + "grad_norm": 1.1558104753494263, + "kl": 3.9140625, + "learning_rate": 1.6015461045510626e-07, + "loss": 0.2275, + "num_tokens": 1335158852.0, + "reward": 1.95751953125, + "reward_std": 0.5034546852111816, + "rewards/accuracy_reward/mean": 0.138671875, + "rewards/accuracy_reward/std": 0.34594178199768066, + "rewards/format_reward/mean": 0.873046875, + "rewards/format_reward/std": 0.33324605226516724, + "rewards/tag_count_reward/mean": 0.94580078125, + "rewards/tag_count_reward/std": 0.15832088887691498, + "step": 2491 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2028.0, + "completions/mean_length": 849.712890625, + "completions/mean_terminated_length": 816.0260620117188, + "completions/min_length": 197.0, + "completions/min_terminated_length": 197.0, + "epoch": 0.8507297089698729, + "grad_norm": 1.2182904481887817, + "kl": 5.984375, + "learning_rate": 1.598871098045036e-07, + "loss": 0.3652, + "num_tokens": 1335678353.0, + "reward": 1.88720703125, + "reward_std": 0.6029765009880066, + "rewards/accuracy_reward/mean": 0.1171875, + "rewards/accuracy_reward/std": 0.32195815443992615, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.92626953125, + "rewards/tag_count_reward/std": 0.19859671592712402, + "step": 2492 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1916.0, + "completions/mean_length": 805.84765625, + "completions/mean_terminated_length": 760.5870361328125, + "completions/min_length": 158.0, + "completions/min_terminated_length": 158.0, + "epoch": 0.8510710932832636, + "grad_norm": 1.2753212451934814, + "kl": 6.203125, + "learning_rate": 1.5962016284779578e-07, + "loss": 0.3772, + "num_tokens": 1336165315.0, + "reward": 1.7998046875, + "reward_std": 0.4820001721382141, + "rewards/accuracy_reward/mean": 0.013671875, + "rewards/accuracy_reward/std": 0.1162383034825325, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.9208984375, + "rewards/tag_count_reward/std": 0.20712071657180786, + "step": 2493 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1977.0, + "completions/mean_length": 807.736328125, + "completions/mean_terminated_length": 783.0299072265625, + "completions/min_length": 122.0, + "completions/min_terminated_length": 122.0, + "epoch": 0.8514124775966544, + "grad_norm": 0.8199345469474792, + "kl": 5.203125, + "learning_rate": 1.5935376996386552e-07, + "loss": 0.3086, + "num_tokens": 1336655564.0, + "reward": 1.90576171875, + "reward_std": 0.5557575225830078, + "rewards/accuracy_reward/mean": 0.1270161271095276, + "rewards/accuracy_reward/std": 0.33332720398902893, + "rewards/format_reward/mean": 0.857421875, + "rewards/format_reward/std": 0.3499840497970581, + "rewards/tag_count_reward/mean": 0.92529296875, + "rewards/tag_count_reward/std": 0.19323164224624634, + "step": 2494 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 800.568359375, + "completions/mean_terminated_length": 736.5318603515625, + "completions/min_length": 166.0, + "completions/min_terminated_length": 166.0, + "epoch": 0.8517538619100452, + "grad_norm": 1.5452642440795898, + "kl": 7.2421875, + "learning_rate": 1.590879315308086e-07, + "loss": 0.4371, + "num_tokens": 1337137519.0, + "reward": 1.8212890625, + "reward_std": 0.6151126623153687, + "rewards/accuracy_reward/mean": 0.0927419364452362, + "rewards/accuracy_reward/std": 0.2903633117675781, + "rewards/format_reward/mean": 0.82421875, + "rewards/format_reward/std": 0.3810062110424042, + "rewards/tag_count_reward/mean": 0.9072265625, + "rewards/tag_count_reward/std": 0.226512148976326, + "step": 2495 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1915.0, + "completions/mean_length": 835.091796875, + "completions/mean_terminated_length": 785.7865600585938, + "completions/min_length": 131.0, + "completions/min_terminated_length": 131.0, + "epoch": 0.852095246223436, + "grad_norm": 1.4080651998519897, + "kl": 5.32421875, + "learning_rate": 1.5882264792593397e-07, + "loss": 0.3321, + "num_tokens": 1337647918.0, + "reward": 1.91796875, + "reward_std": 0.4931030571460724, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.880859375, + "rewards/format_reward/std": 0.32427072525024414, + "rewards/tag_count_reward/mean": 0.939453125, + "rewards/tag_count_reward/std": 0.17485272884368896, + "step": 2496 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1990.0, + "completions/mean_length": 870.51171875, + "completions/mean_terminated_length": 830.07275390625, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "epoch": 0.8524366305368268, + "grad_norm": 2.8357620239257812, + "kl": 5.578125, + "learning_rate": 1.5855791952576342e-07, + "loss": 0.3538, + "num_tokens": 1338170036.0, + "reward": 1.837890625, + "reward_std": 0.5277907252311707, + "rewards/accuracy_reward/mean": 0.060483869165182114, + "rewards/accuracy_reward/std": 0.2386218160390854, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.923828125, + "rewards/tag_count_reward/std": 0.19549374282360077, + "step": 2497 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1882.0, + "completions/mean_length": 854.41015625, + "completions/mean_terminated_length": 800.8203735351562, + "completions/min_length": 159.0, + "completions/min_terminated_length": 159.0, + "epoch": 0.8527780148502176, + "grad_norm": 1.2506555318832397, + "kl": 6.71875, + "learning_rate": 1.582937467060302e-07, + "loss": 0.4093, + "num_tokens": 1338696598.0, + "reward": 1.77880859375, + "reward_std": 0.5462380647659302, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.82421875, + "rewards/format_reward/std": 0.3810062110424042, + "rewards/tag_count_reward/mean": 0.91357421875, + "rewards/tag_count_reward/std": 0.20386749505996704, + "step": 2498 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.064453125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 906.654296875, + "completions/mean_terminated_length": 828.0230102539062, + "completions/min_length": 144.0, + "completions/min_terminated_length": 144.0, + "epoch": 0.8531193991636085, + "grad_norm": 2.58473539352417, + "kl": 7.6328125, + "learning_rate": 1.5803012984167963e-07, + "loss": 0.4522, + "num_tokens": 1339242885.0, + "reward": 1.80859375, + "reward_std": 0.5806085467338562, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.826171875, + "rewards/format_reward/std": 0.3793322443962097, + "rewards/tag_count_reward/mean": 0.904296875, + "rewards/tag_count_reward/std": 0.2181157022714615, + "step": 2499 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.056640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1982.0, + "completions/mean_length": 931.302734375, + "completions/mean_terminated_length": 864.2546997070312, + "completions/min_length": 108.0, + "completions/min_terminated_length": 108.0, + "epoch": 0.8534607834769993, + "grad_norm": 1.5278397798538208, + "kl": 7.0703125, + "learning_rate": 1.5776706930686738e-07, + "loss": 0.4381, + "num_tokens": 1339805136.0, + "reward": 1.8017578125, + "reward_std": 0.5659550428390503, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.83203125, + "rewards/format_reward/std": 0.374204158782959, + "rewards/tag_count_reward/mean": 0.9169921875, + "rewards/tag_count_reward/std": 0.21028900146484375, + "step": 2500 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1789.0, + "completions/mean_length": 842.298828125, + "completions/mean_terminated_length": 805.909423828125, + "completions/min_length": 99.0, + "completions/min_terminated_length": 99.0, + "epoch": 0.85380216779039, + "grad_norm": 2.8222107887268066, + "kl": 5.94140625, + "learning_rate": 1.5750456547495995e-07, + "loss": 0.4095, + "num_tokens": 1340313097.0, + "reward": 1.8447265625, + "reward_std": 0.4956468939781189, + "rewards/accuracy_reward/mean": 0.032258063554763794, + "rewards/accuracy_reward/std": 0.17686307430267334, + "rewards/format_reward/mean": 0.876953125, + "rewards/format_reward/std": 0.32881227135658264, + "rewards/tag_count_reward/mean": 0.9365234375, + "rewards/tag_count_reward/std": 0.18339595198631287, + "step": 2501 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1976.0, + "completions/mean_length": 848.896484375, + "completions/mean_terminated_length": 787.3408813476562, + "completions/min_length": 110.0, + "completions/min_terminated_length": 110.0, + "epoch": 0.8541435521037808, + "grad_norm": 1.4963473081588745, + "kl": 6.296875, + "learning_rate": 1.572426187185334e-07, + "loss": 0.3906, + "num_tokens": 1340827364.0, + "reward": 1.82080078125, + "reward_std": 0.5283799171447754, + "rewards/accuracy_reward/mean": 0.0463709682226181, + "rewards/accuracy_reward/std": 0.21049949526786804, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.92431640625, + "rewards/tag_count_reward/std": 0.1984763890504837, + "step": 2502 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1818.0, + "completions/mean_length": 784.9765625, + "completions/mean_terminated_length": 738.9555053710938, + "completions/min_length": 157.0, + "completions/min_terminated_length": 157.0, + "epoch": 0.8544849364171716, + "grad_norm": 2.6061666011810303, + "kl": 5.6484375, + "learning_rate": 1.5698122940937325e-07, + "loss": 0.3959, + "num_tokens": 1341301432.0, + "reward": 1.8818359375, + "reward_std": 0.49414151906967163, + "rewards/accuracy_reward/mean": 0.06854838877916336, + "rewards/accuracy_reward/std": 0.25293970108032227, + "rewards/format_reward/mean": 0.87890625, + "rewards/format_reward/std": 0.3265552520751953, + "rewards/tag_count_reward/mean": 0.9365234375, + "rewards/tag_count_reward/std": 0.18604448437690735, + "step": 2503 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1990.0, + "completions/mean_length": 821.587890625, + "completions/mean_terminated_length": 789.6372680664062, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "epoch": 0.8548263207305624, + "grad_norm": 1.573183298110962, + "kl": 5.01953125, + "learning_rate": 1.5672039791847385e-07, + "loss": 0.2964, + "num_tokens": 1341791429.0, + "reward": 1.83984375, + "reward_std": 0.4401506185531616, + "rewards/accuracy_reward/mean": 0.015625, + "rewards/accuracy_reward/std": 0.12414088100194931, + "rewards/format_reward/mean": 0.8828125, + "rewards/format_reward/std": 0.32195815443992615, + "rewards/tag_count_reward/mean": 0.94140625, + "rewards/tag_count_reward/std": 0.17199864983558655, + "step": 2504 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1984.0, + "completions/mean_length": 841.2109375, + "completions/mean_terminated_length": 797.2388916015625, + "completions/min_length": 167.0, + "completions/min_terminated_length": 167.0, + "epoch": 0.8551677050439532, + "grad_norm": 2.2540669441223145, + "kl": 6.12890625, + "learning_rate": 1.5646012461603773e-07, + "loss": 0.4215, + "num_tokens": 1342291985.0, + "reward": 1.87255859375, + "reward_std": 0.5276371240615845, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.93310546875, + "rewards/tag_count_reward/std": 0.18583056330680847, + "step": 2505 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 825.771484375, + "completions/mean_terminated_length": 798.9360961914062, + "completions/min_length": 78.0, + "completions/min_terminated_length": 78.0, + "epoch": 0.855509089357344, + "grad_norm": 1.54689621925354, + "kl": 4.474609375, + "learning_rate": 1.5620040987147536e-07, + "loss": 0.279, + "num_tokens": 1342792396.0, + "reward": 1.91552734375, + "reward_std": 0.4751802086830139, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.900390625, + "rewards/format_reward/std": 0.29977133870124817, + "rewards/tag_count_reward/mean": 0.93896484375, + "rewards/tag_count_reward/std": 0.1825571358203888, + "step": 2506 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1998.0, + "completions/mean_length": 796.3828125, + "completions/mean_terminated_length": 776.5159301757812, + "completions/min_length": 112.0, + "completions/min_terminated_length": 112.0, + "epoch": 0.8558504736707349, + "grad_norm": 1.6224844455718994, + "kl": 4.1015625, + "learning_rate": 1.559412540534043e-07, + "loss": 0.2462, + "num_tokens": 1343279280.0, + "reward": 1.900390625, + "reward_std": 0.4144550859928131, + "rewards/accuracy_reward/mean": 0.03427419438958168, + "rewards/accuracy_reward/std": 0.18211629986763, + "rewards/format_reward/mean": 0.91015625, + "rewards/format_reward/std": 0.2862374484539032, + "rewards/tag_count_reward/mean": 0.95703125, + "rewards/tag_count_reward/std": 0.15438589453697205, + "step": 2507 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2000.0, + "completions/mean_length": 780.849609375, + "completions/mean_terminated_length": 753.0279541015625, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "epoch": 0.8561918579841257, + "grad_norm": 0.9053124785423279, + "kl": 4.8203125, + "learning_rate": 1.5568265752964865e-07, + "loss": 0.2992, + "num_tokens": 1343756947.0, + "reward": 1.9140625, + "reward_std": 0.4191058874130249, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.912109375, + "rewards/format_reward/std": 0.2834126651287079, + "rewards/tag_count_reward/mean": 0.951171875, + "rewards/tag_count_reward/std": 0.17078326642513275, + "step": 2508 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1950.0, + "completions/mean_length": 796.09375, + "completions/mean_terminated_length": 778.7406005859375, + "completions/min_length": 151.0, + "completions/min_terminated_length": 151.0, + "epoch": 0.8565332422975165, + "grad_norm": 4.397088527679443, + "kl": 4.3828125, + "learning_rate": 1.5542462066723912e-07, + "loss": 0.2746, + "num_tokens": 1344241939.0, + "reward": 1.9208984375, + "reward_std": 0.38755735754966736, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.91796875, + "rewards/format_reward/std": 0.2746807038784027, + "rewards/tag_count_reward/mean": 0.9560546875, + "rewards/tag_count_reward/std": 0.1517105996608734, + "step": 2509 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1988.0, + "completions/mean_length": 865.845703125, + "completions/mean_terminated_length": 830.1669921875, + "completions/min_length": 64.0, + "completions/min_terminated_length": 64.0, + "epoch": 0.8568746266109072, + "grad_norm": 0.8987749814987183, + "kl": 5.4609375, + "learning_rate": 1.551671438324116e-07, + "loss": 0.3491, + "num_tokens": 1344768708.0, + "reward": 1.9482421875, + "reward_std": 0.46120744943618774, + "rewards/accuracy_reward/mean": 0.103515625, + "rewards/accuracy_reward/std": 0.30492907762527466, + "rewards/format_reward/mean": 0.8984375, + "rewards/format_reward/std": 0.30236753821372986, + "rewards/tag_count_reward/mean": 0.9462890625, + "rewards/tag_count_reward/std": 0.17288248240947723, + "step": 2510 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1776.0, + "completions/mean_length": 824.564453125, + "completions/mean_terminated_length": 797.7025756835938, + "completions/min_length": 178.0, + "completions/min_terminated_length": 178.0, + "epoch": 0.857216010924298, + "grad_norm": 1.2078722715377808, + "kl": 5.28125, + "learning_rate": 1.549102273906076e-07, + "loss": 0.3368, + "num_tokens": 1345279525.0, + "reward": 1.91015625, + "reward_std": 0.43426308035850525, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.908203125, + "rewards/format_reward/std": 0.289021372795105, + "rewards/tag_count_reward/mean": 0.951171875, + "rewards/tag_count_reward/std": 0.16271483898162842, + "step": 2511 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 835.6640625, + "completions/mean_terminated_length": 791.4898681640625, + "completions/min_length": 152.0, + "completions/min_terminated_length": 152.0, + "epoch": 0.8575573952376888, + "grad_norm": 2.947654962539673, + "kl": 7.4765625, + "learning_rate": 1.5465387170647284e-07, + "loss": 0.4508, + "num_tokens": 1345780601.0, + "reward": 1.88427734375, + "reward_std": 0.47508326172828674, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.880859375, + "rewards/format_reward/std": 0.32427072525024414, + "rewards/tag_count_reward/mean": 0.93505859375, + "rewards/tag_count_reward/std": 0.19169752299785614, + "step": 2512 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1844.0, + "completions/mean_length": 807.88671875, + "completions/mean_terminated_length": 773.0240478515625, + "completions/min_length": 62.0, + "completions/min_terminated_length": 62.0, + "epoch": 0.8578987795510796, + "grad_norm": 1.9713817834854126, + "kl": 5.75390625, + "learning_rate": 1.5439807714385747e-07, + "loss": 0.3325, + "num_tokens": 1346274911.0, + "reward": 1.93212890625, + "reward_std": 0.5016872882843018, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.892578125, + "rewards/format_reward/std": 0.30995169281959534, + "rewards/tag_count_reward/mean": 0.94970703125, + "rewards/tag_count_reward/std": 0.16037173569202423, + "step": 2513 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 832.283203125, + "completions/mean_terminated_length": 805.5908203125, + "completions/min_length": 161.0, + "completions/min_terminated_length": 161.0, + "epoch": 0.8582401638644704, + "grad_norm": 1.8401319980621338, + "kl": 6.2265625, + "learning_rate": 1.54142844065815e-07, + "loss": 0.3435, + "num_tokens": 1346782384.0, + "reward": 1.9248046875, + "reward_std": 0.42352402210235596, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.90625, + "rewards/format_reward/std": 0.29176566004753113, + "rewards/tag_count_reward/mean": 0.9560546875, + "rewards/tag_count_reward/std": 0.14679372310638428, + "step": 2514 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1917.0, + "completions/mean_length": 768.3671875, + "completions/mean_terminated_length": 745.4711303710938, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "epoch": 0.8585815481778613, + "grad_norm": 1.9592394828796387, + "kl": 6.00390625, + "learning_rate": 1.5388817283460205e-07, + "loss": 0.359, + "num_tokens": 1347251980.0, + "reward": 1.92138671875, + "reward_std": 0.42750340700149536, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.90234375, + "rewards/format_reward/std": 0.29713961482048035, + "rewards/tag_count_reward/mean": 0.94677734375, + "rewards/tag_count_reward/std": 0.1719701737165451, + "step": 2515 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1955.0, + "completions/mean_length": 821.07421875, + "completions/mean_terminated_length": 794.1357421875, + "completions/min_length": 121.0, + "completions/min_terminated_length": 121.0, + "epoch": 0.8589229324912521, + "grad_norm": 2.8998708724975586, + "kl": 5.7265625, + "learning_rate": 1.5363406381167798e-07, + "loss": 0.3285, + "num_tokens": 1347750978.0, + "reward": 1.88525390625, + "reward_std": 0.3968276083469391, + "rewards/accuracy_reward/mean": 0.033203125, + "rewards/accuracy_reward/std": 0.17934183776378632, + "rewards/format_reward/mean": 0.904296875, + "rewards/format_reward/std": 0.2944713830947876, + "rewards/tag_count_reward/mean": 0.94775390625, + "rewards/tag_count_reward/std": 0.17155851423740387, + "step": 2516 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1952.0, + "completions/mean_length": 799.5859375, + "completions/mean_terminated_length": 754.09716796875, + "completions/min_length": 180.0, + "completions/min_terminated_length": 180.0, + "epoch": 0.8592643168046429, + "grad_norm": 0.846878170967102, + "kl": 6.20703125, + "learning_rate": 1.533805173577039e-07, + "loss": 0.4029, + "num_tokens": 1348235790.0, + "reward": 1.9208984375, + "reward_std": 0.46979883313179016, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.90234375, + "rewards/format_reward/std": 0.29713961482048035, + "rewards/tag_count_reward/mean": 0.9482421875, + "rewards/tag_count_reward/std": 0.170634925365448, + "step": 2517 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2031.0, + "completions/mean_length": 846.21875, + "completions/mean_terminated_length": 804.9454956054688, + "completions/min_length": 71.0, + "completions/min_terminated_length": 71.0, + "epoch": 0.8596057011180336, + "grad_norm": 2.724597215652466, + "kl": 6.79296875, + "learning_rate": 1.531275338325429e-07, + "loss": 0.3978, + "num_tokens": 1348749358.0, + "reward": 1.90380859375, + "reward_std": 0.5706411600112915, + "rewards/accuracy_reward/mean": 0.12890625, + "rewards/accuracy_reward/std": 0.33542385697364807, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.92724609375, + "rewards/tag_count_reward/std": 0.19207598268985748, + "step": 2518 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1873.0, + "completions/mean_length": 785.5625, + "completions/mean_terminated_length": 755.2640380859375, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "epoch": 0.8599470854314244, + "grad_norm": 2.1348705291748047, + "kl": 5.8515625, + "learning_rate": 1.528751135952585e-07, + "loss": 0.3575, + "num_tokens": 1349235422.0, + "reward": 1.923828125, + "reward_std": 0.4424586892127991, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.8984375, + "rewards/format_reward/std": 0.30236753821372986, + "rewards/tag_count_reward/mean": 0.943359375, + "rewards/tag_count_reward/std": 0.17754259705543518, + "step": 2519 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1990.0, + "completions/mean_length": 731.671875, + "completions/mean_terminated_length": 710.77783203125, + "completions/min_length": 184.0, + "completions/min_terminated_length": 184.0, + "epoch": 0.8602884697448152, + "grad_norm": 1.0865064859390259, + "kl": 3.98046875, + "learning_rate": 1.5262325700411534e-07, + "loss": 0.2623, + "num_tokens": 1349688662.0, + "reward": 1.99267578125, + "reward_std": 0.38150593638420105, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.923828125, + "rewards/format_reward/std": 0.26553234457969666, + "rewards/tag_count_reward/mean": 0.96337890625, + "rewards/tag_count_reward/std": 0.13454900681972504, + "step": 2520 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1920.0, + "completions/mean_length": 770.08203125, + "completions/mean_terminated_length": 731.5130615234375, + "completions/min_length": 143.0, + "completions/min_terminated_length": 143.0, + "epoch": 0.860629854058206, + "grad_norm": 1.1588002443313599, + "kl": 6.4140625, + "learning_rate": 1.5237196441657767e-07, + "loss": 0.433, + "num_tokens": 1350163408.0, + "reward": 1.9150390625, + "reward_std": 0.5046082735061646, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.8828125, + "rewards/format_reward/std": 0.32195815443992615, + "rewards/tag_count_reward/mean": 0.9423828125, + "rewards/tag_count_reward/std": 0.1709035038948059, + "step": 2521 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2014.0, + "completions/mean_length": 832.359375, + "completions/mean_terminated_length": 777.779541015625, + "completions/min_length": 135.0, + "completions/min_terminated_length": 135.0, + "epoch": 0.8609712383715968, + "grad_norm": 2.694650650024414, + "kl": 6.46875, + "learning_rate": 1.5212123618930924e-07, + "loss": 0.3736, + "num_tokens": 1350670904.0, + "reward": 1.84033203125, + "reward_std": 0.522647500038147, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.92822265625, + "rewards/tag_count_reward/std": 0.19244377315044403, + "step": 2522 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 792.345703125, + "completions/mean_terminated_length": 757.046142578125, + "completions/min_length": 128.0, + "completions/min_terminated_length": 128.0, + "epoch": 0.8613126226849876, + "grad_norm": 1.2514175176620483, + "kl": 6.6015625, + "learning_rate": 1.518710726781731e-07, + "loss": 0.4233, + "num_tokens": 1351153129.0, + "reward": 1.90869140625, + "reward_std": 0.5649210810661316, + "rewards/accuracy_reward/mean": 0.111328125, + "rewards/accuracy_reward/std": 0.31484565138816833, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.93212890625, + "rewards/tag_count_reward/std": 0.1834864467382431, + "step": 2523 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1947.0, + "completions/mean_length": 790.642578125, + "completions/mean_terminated_length": 750.0826416015625, + "completions/min_length": 154.0, + "completions/min_terminated_length": 154.0, + "epoch": 0.8616540069983785, + "grad_norm": 1.3608920574188232, + "kl": 6.1328125, + "learning_rate": 1.5162147423823043e-07, + "loss": 0.3918, + "num_tokens": 1351638034.0, + "reward": 1.91455078125, + "reward_std": 0.577094316482544, + "rewards/accuracy_reward/mean": 0.130859375, + "rewards/accuracy_reward/std": 0.33757632970809937, + "rewards/format_reward/mean": 0.857421875, + "rewards/format_reward/std": 0.3499840497970581, + "rewards/tag_count_reward/mean": 0.92626953125, + "rewards/tag_count_reward/std": 0.19674043357372284, + "step": 2524 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 821.892578125, + "completions/mean_terminated_length": 779.7838745117188, + "completions/min_length": 131.0, + "completions/min_terminated_length": 131.0, + "epoch": 0.8619953913117693, + "grad_norm": 1.9353266954421997, + "kl": 5.9375, + "learning_rate": 1.5137244122374076e-07, + "loss": 0.362, + "num_tokens": 1352131995.0, + "reward": 1.853515625, + "reward_std": 0.5325428247451782, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.93359375, + "rewards/tag_count_reward/std": 0.17965060472488403, + "step": 2525 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1986.0, + "completions/mean_length": 809.013671875, + "completions/mean_terminated_length": 766.462646484375, + "completions/min_length": 131.0, + "completions/min_terminated_length": 131.0, + "epoch": 0.86233677562516, + "grad_norm": 0.9738795161247253, + "kl": 6.203125, + "learning_rate": 1.5112397398816076e-07, + "loss": 0.407, + "num_tokens": 1352626498.0, + "reward": 1.83642578125, + "reward_std": 0.559461236000061, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.91845703125, + "rewards/tag_count_reward/std": 0.2022770643234253, + "step": 2526 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1923.0, + "completions/mean_length": 767.6484375, + "completions/mean_terminated_length": 734.2926025390625, + "completions/min_length": 76.0, + "completions/min_terminated_length": 76.0, + "epoch": 0.8626781599385508, + "grad_norm": 3.458853244781494, + "kl": 5.58984375, + "learning_rate": 1.508760728841442e-07, + "loss": 0.4151, + "num_tokens": 1353100606.0, + "reward": 1.82470703125, + "reward_std": 0.4834443926811218, + "rewards/accuracy_reward/mean": 0.026209676638245583, + "rewards/accuracy_reward/std": 0.1599196344614029, + "rewards/format_reward/mean": 0.8671875, + "rewards/format_reward/std": 0.33970388770103455, + "rewards/tag_count_reward/mean": 0.93212890625, + "rewards/tag_count_reward/std": 0.1906779557466507, + "step": 2527 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1841.0, + "completions/mean_length": 797.880859375, + "completions/mean_terminated_length": 752.3299560546875, + "completions/min_length": 131.0, + "completions/min_terminated_length": 131.0, + "epoch": 0.8630195442519416, + "grad_norm": 2.578061103820801, + "kl": 5.765625, + "learning_rate": 1.506287382635415e-07, + "loss": 0.3849, + "num_tokens": 1353604833.0, + "reward": 1.8681640625, + "reward_std": 0.5036935210227966, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.9384765625, + "rewards/tag_count_reward/std": 0.17590700089931488, + "step": 2528 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1767.0, + "completions/mean_length": 798.412109375, + "completions/mean_terminated_length": 747.6158447265625, + "completions/min_length": 152.0, + "completions/min_terminated_length": 152.0, + "epoch": 0.8633609285653324, + "grad_norm": 3.326608419418335, + "kl": 4.640625, + "learning_rate": 1.503819704773987e-07, + "loss": 0.3471, + "num_tokens": 1354089796.0, + "reward": 1.90234375, + "reward_std": 0.4647481143474579, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.892578125, + "rewards/format_reward/std": 0.30995169281959534, + "rewards/tag_count_reward/mean": 0.94921875, + "rewards/tag_count_reward/std": 0.16658000648021698, + "step": 2529 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1917.0, + "completions/mean_length": 775.318359375, + "completions/mean_terminated_length": 715.4580688476562, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "epoch": 0.8637023128787232, + "grad_norm": 1.401532769203186, + "kl": 7.0234375, + "learning_rate": 1.501357698759578e-07, + "loss": 0.4558, + "num_tokens": 1354558583.0, + "reward": 1.86279296875, + "reward_std": 0.5681871771812439, + "rewards/accuracy_reward/mean": 0.08467742055654526, + "rewards/accuracy_reward/std": 0.278682142496109, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.92724609375, + "rewards/tag_count_reward/std": 0.19523389637470245, + "step": 2530 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1914.0, + "completions/mean_length": 782.685546875, + "completions/mean_terminated_length": 741.8689575195312, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "epoch": 0.864043697192114, + "grad_norm": 2.5293610095977783, + "kl": 5.68359375, + "learning_rate": 1.498901368086553e-07, + "loss": 0.372, + "num_tokens": 1355034646.0, + "reward": 1.8740234375, + "reward_std": 0.5300194621086121, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.9345703125, + "rewards/tag_count_reward/std": 0.17864517867565155, + "step": 2531 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1889.0, + "completions/mean_length": 847.482421875, + "completions/mean_terminated_length": 791.016357421875, + "completions/min_length": 149.0, + "completions/min_terminated_length": 149.0, + "epoch": 0.8643850815055049, + "grad_norm": 1.91965651512146, + "kl": 6.7578125, + "learning_rate": 1.4964507162412268e-07, + "loss": 0.4391, + "num_tokens": 1355549549.0, + "reward": 1.84375, + "reward_std": 0.5757134556770325, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.921875, + "rewards/tag_count_reward/std": 0.20512036979198456, + "step": 2532 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1939.0, + "completions/mean_length": 799.568359375, + "completions/mean_terminated_length": 746.1731567382812, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "epoch": 0.8647264658188957, + "grad_norm": 1.4131958484649658, + "kl": 7.1328125, + "learning_rate": 1.4940057467018482e-07, + "loss": 0.4115, + "num_tokens": 1356031168.0, + "reward": 1.81689453125, + "reward_std": 0.5381094217300415, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.91845703125, + "rewards/tag_count_reward/std": 0.20823590457439423, + "step": 2533 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1823.0, + "completions/mean_length": 772.65234375, + "completions/mean_terminated_length": 747.2470092773438, + "completions/min_length": 82.0, + "completions/min_terminated_length": 82.0, + "epoch": 0.8650678501322864, + "grad_norm": 1.1551711559295654, + "kl": 4.83203125, + "learning_rate": 1.491566462938605e-07, + "loss": 0.2803, + "num_tokens": 1356505134.0, + "reward": 1.9287109375, + "reward_std": 0.4677783250808716, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.904296875, + "rewards/format_reward/std": 0.2944713830947876, + "rewards/tag_count_reward/mean": 0.9482421875, + "rewards/tag_count_reward/std": 0.170634925365448, + "step": 2534 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1981.0, + "completions/mean_length": 793.908203125, + "completions/mean_terminated_length": 763.81005859375, + "completions/min_length": 181.0, + "completions/min_terminated_length": 181.0, + "epoch": 0.8654092344456772, + "grad_norm": 1.8901922702789307, + "kl": 6.2421875, + "learning_rate": 1.489132868413617e-07, + "loss": 0.4336, + "num_tokens": 1356998495.0, + "reward": 1.86669921875, + "reward_std": 0.48085707426071167, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.87890625, + "rewards/format_reward/std": 0.3265552520751953, + "rewards/tag_count_reward/mean": 0.94287109375, + "rewards/tag_count_reward/std": 0.17213678359985352, + "step": 2535 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1860.0, + "completions/mean_length": 794.125, + "completions/mean_terminated_length": 745.8012084960938, + "completions/min_length": 152.0, + "completions/min_terminated_length": 152.0, + "epoch": 0.865750618759068, + "grad_norm": 0.9839769601821899, + "kl": 7.3359375, + "learning_rate": 1.4867049665809232e-07, + "loss": 0.4569, + "num_tokens": 1357482591.0, + "reward": 1.86962890625, + "reward_std": 0.4972394108772278, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.875, + "rewards/format_reward/std": 0.3310423493385315, + "rewards/tag_count_reward/mean": 0.93017578125, + "rewards/tag_count_reward/std": 0.19630283117294312, + "step": 2536 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1973.0, + "completions/mean_length": 773.451171875, + "completions/mean_terminated_length": 737.6204833984375, + "completions/min_length": 114.0, + "completions/min_terminated_length": 114.0, + "epoch": 0.8660920030724588, + "grad_norm": 0.9966095685958862, + "kl": 6.2734375, + "learning_rate": 1.4842827608864886e-07, + "loss": 0.3642, + "num_tokens": 1357958662.0, + "reward": 1.9443359375, + "reward_std": 0.5586145520210266, + "rewards/accuracy_reward/mean": 0.1391129046678543, + "rewards/accuracy_reward/std": 0.3464137017726898, + "rewards/format_reward/mean": 0.873046875, + "rewards/format_reward/std": 0.33324605226516724, + "rewards/tag_count_reward/mean": 0.9365234375, + "rewards/tag_count_reward/std": 0.1820572316646576, + "step": 2537 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 715.365234375, + "completions/mean_terminated_length": 686.1057739257812, + "completions/min_length": 88.0, + "completions/min_terminated_length": 88.0, + "epoch": 0.8664333873858496, + "grad_norm": 1.615759015083313, + "kl": 7.0390625, + "learning_rate": 1.48186625476819e-07, + "loss": 0.4503, + "num_tokens": 1358404289.0, + "reward": 1.94287109375, + "reward_std": 0.5075457096099854, + "rewards/accuracy_reward/mean": 0.11491935700178146, + "rewards/accuracy_reward/std": 0.3192465901374817, + "rewards/format_reward/mean": 0.892578125, + "rewards/format_reward/std": 0.30995169281959534, + "rewards/tag_count_reward/mean": 0.93896484375, + "rewards/tag_count_reward/std": 0.18322588503360748, + "step": 2538 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1987.0, + "completions/mean_length": 803.2890625, + "completions/mean_terminated_length": 781.0178833007812, + "completions/min_length": 170.0, + "completions/min_terminated_length": 170.0, + "epoch": 0.8667747716992404, + "grad_norm": 2.0541937351226807, + "kl": 4.6796875, + "learning_rate": 1.4794554516558166e-07, + "loss": 0.3296, + "num_tokens": 1358893781.0, + "reward": 1.9755859375, + "reward_std": 0.4012356698513031, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.923828125, + "rewards/format_reward/std": 0.26553234457969666, + "rewards/tag_count_reward/mean": 0.9560546875, + "rewards/tag_count_reward/std": 0.16185224056243896, + "step": 2539 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 738.998046875, + "completions/mean_terminated_length": 704.8958129882812, + "completions/min_length": 135.0, + "completions/min_terminated_length": 135.0, + "epoch": 0.8671161560126313, + "grad_norm": 1.09844970703125, + "kl": 4.75, + "learning_rate": 1.477050354971061e-07, + "loss": 0.2915, + "num_tokens": 1359347220.0, + "reward": 1.9404296875, + "reward_std": 0.4506056308746338, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.904296875, + "rewards/format_reward/std": 0.2944713830947876, + "rewards/tag_count_reward/mean": 0.9560546875, + "rewards/tag_count_reward/std": 0.14845077693462372, + "step": 2540 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1884.0, + "completions/mean_length": 747.431640625, + "completions/mean_terminated_length": 726.7877197265625, + "completions/min_length": 131.0, + "completions/min_terminated_length": 131.0, + "epoch": 0.8674575403260221, + "grad_norm": 1.7966375350952148, + "kl": 4.92578125, + "learning_rate": 1.474650968127518e-07, + "loss": 0.3323, + "num_tokens": 1359811713.0, + "reward": 1.9609375, + "reward_std": 0.3912133574485779, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.923828125, + "rewards/format_reward/std": 0.26553234457969666, + "rewards/tag_count_reward/mean": 0.9609375, + "rewards/tag_count_reward/std": 0.14899368584156036, + "step": 2541 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1926.0, + "completions/mean_length": 819.41015625, + "completions/mean_terminated_length": 787.40283203125, + "completions/min_length": 119.0, + "completions/min_terminated_length": 119.0, + "epoch": 0.8677989246394128, + "grad_norm": 1.3843644857406616, + "kl": 4.751953125, + "learning_rate": 1.4722572945306812e-07, + "loss": 0.3132, + "num_tokens": 1360319683.0, + "reward": 1.91748046875, + "reward_std": 0.41918283700942993, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.904296875, + "rewards/format_reward/std": 0.2944713830947876, + "rewards/tag_count_reward/mean": 0.95458984375, + "rewards/tag_count_reward/std": 0.1548733115196228, + "step": 2542 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2012.0, + "completions/mean_length": 726.431640625, + "completions/mean_terminated_length": 697.4151611328125, + "completions/min_length": 38.0, + "completions/min_terminated_length": 38.0, + "epoch": 0.8681403089528036, + "grad_norm": 1.3754132986068726, + "kl": 5.34765625, + "learning_rate": 1.4698693375779296e-07, + "loss": 0.3411, + "num_tokens": 1360779072.0, + "reward": 1.96826171875, + "reward_std": 0.43035122752189636, + "rewards/accuracy_reward/mean": 0.103515625, + "rewards/accuracy_reward/std": 0.30492907762527466, + "rewards/format_reward/mean": 0.912109375, + "rewards/format_reward/std": 0.2834126651287079, + "rewards/tag_count_reward/mean": 0.95263671875, + "rewards/tag_count_reward/std": 0.16352249681949615, + "step": 2543 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1945.0, + "completions/mean_length": 782.02734375, + "completions/mean_terminated_length": 751.64404296875, + "completions/min_length": 49.0, + "completions/min_terminated_length": 49.0, + "epoch": 0.8684816932661944, + "grad_norm": 0.908316969871521, + "kl": 4.83984375, + "learning_rate": 1.4674871006585326e-07, + "loss": 0.318, + "num_tokens": 1361261438.0, + "reward": 1.91162109375, + "reward_std": 0.4011594355106354, + "rewards/accuracy_reward/mean": 0.03125, + "rewards/accuracy_reward/std": 0.17416280508041382, + "rewards/format_reward/mean": 0.92578125, + "rewards/format_reward/std": 0.2623828947544098, + "rewards/tag_count_reward/mean": 0.95458984375, + "rewards/tag_count_reward/std": 0.16407670080661774, + "step": 2544 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1882.0, + "completions/mean_length": 814.658203125, + "completions/mean_terminated_length": 774.8729858398438, + "completions/min_length": 148.0, + "completions/min_terminated_length": 148.0, + "epoch": 0.8688230775795852, + "grad_norm": 0.8029277324676514, + "kl": 6.55078125, + "learning_rate": 1.46511058715364e-07, + "loss": 0.4211, + "num_tokens": 1361749663.0, + "reward": 1.8876953125, + "reward_std": 0.39295393228530884, + "rewards/accuracy_reward/mean": 0.021484375, + "rewards/accuracy_reward/std": 0.14513419568538666, + "rewards/format_reward/mean": 0.9140625, + "rewards/format_reward/std": 0.28054583072662354, + "rewards/tag_count_reward/mean": 0.9521484375, + "rewards/tag_count_reward/std": 0.16524095833301544, + "step": 2545 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1915.0, + "completions/mean_length": 778.298828125, + "completions/mean_terminated_length": 750.421142578125, + "completions/min_length": 179.0, + "completions/min_terminated_length": 179.0, + "epoch": 0.869164461892976, + "grad_norm": 1.4204837083816528, + "kl": 5.6171875, + "learning_rate": 1.4627398004362774e-07, + "loss": 0.3371, + "num_tokens": 1362234728.0, + "reward": 1.90966796875, + "reward_std": 0.410476416349411, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.912109375, + "rewards/format_reward/std": 0.2834126651287079, + "rewards/tag_count_reward/mean": 0.95458984375, + "rewards/tag_count_reward/std": 0.15566104650497437, + "step": 2546 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1956.0, + "completions/mean_length": 802.7109375, + "completions/mean_terminated_length": 767.7027587890625, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "epoch": 0.8695058462063668, + "grad_norm": 1.8084684610366821, + "kl": 6.5, + "learning_rate": 1.4603747438713426e-07, + "loss": 0.4016, + "num_tokens": 1362725940.0, + "reward": 1.8994140625, + "reward_std": 0.38757991790771484, + "rewards/accuracy_reward/mean": 0.033203125, + "rewards/accuracy_reward/std": 0.17934183776378632, + "rewards/format_reward/mean": 0.9140625, + "rewards/format_reward/std": 0.28054583072662354, + "rewards/tag_count_reward/mean": 0.9521484375, + "rewards/tag_count_reward/std": 0.16375388205051422, + "step": 2547 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2022.0, + "completions/mean_length": 822.4453125, + "completions/mean_terminated_length": 785.4567260742188, + "completions/min_length": 120.0, + "completions/min_terminated_length": 120.0, + "epoch": 0.8698472305197577, + "grad_norm": 3.6708779335021973, + "kl": 7.1484375, + "learning_rate": 1.458015420815601e-07, + "loss": 0.4196, + "num_tokens": 1363225736.0, + "reward": 1.9091796875, + "reward_std": 0.446837842464447, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.890625, + "rewards/format_reward/std": 0.31241437792778015, + "rewards/tag_count_reward/mean": 0.9423828125, + "rewards/tag_count_reward/std": 0.1813209503889084, + "step": 2548 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1769.0, + "completions/mean_length": 751.98046875, + "completions/mean_terminated_length": 723.5249633789062, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.8701886148331485, + "grad_norm": 1.6291117668151855, + "kl": 4.8203125, + "learning_rate": 1.4556618346176813e-07, + "loss": 0.2759, + "num_tokens": 1363694462.0, + "reward": 1.93994140625, + "reward_std": 0.40668895840644836, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.92578125, + "rewards/format_reward/std": 0.2623828947544098, + "rewards/tag_count_reward/mean": 0.95556640625, + "rewards/tag_count_reward/std": 0.15515686571598053, + "step": 2549 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1962.0, + "completions/mean_length": 850.955078125, + "completions/mean_terminated_length": 802.294677734375, + "completions/min_length": 153.0, + "completions/min_terminated_length": 153.0, + "epoch": 0.8705299991465392, + "grad_norm": 2.863211154937744, + "kl": 8.5390625, + "learning_rate": 1.453313988618067e-07, + "loss": 0.5055, + "num_tokens": 1364210135.0, + "reward": 1.8369140625, + "reward_std": 0.5257569551467896, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.9228515625, + "rewards/tag_count_reward/std": 0.20429649949073792, + "step": 2550 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2001.0, + "completions/mean_length": 764.357421875, + "completions/mean_terminated_length": 722.9495849609375, + "completions/min_length": 133.0, + "completions/min_terminated_length": 133.0, + "epoch": 0.87087138345993, + "grad_norm": 1.5717759132385254, + "kl": 5.95703125, + "learning_rate": 1.4509718861490983e-07, + "loss": 0.386, + "num_tokens": 1364665998.0, + "reward": 1.96923828125, + "reward_std": 0.4320371448993683, + "rewards/accuracy_reward/mean": 0.109375, + "rewards/accuracy_reward/std": 0.31241437792778015, + "rewards/format_reward/mean": 0.91015625, + "rewards/format_reward/std": 0.2862374484539032, + "rewards/tag_count_reward/mean": 0.94970703125, + "rewards/tag_count_reward/std": 0.1692764312028885, + "step": 2551 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1981.0, + "completions/mean_length": 744.84765625, + "completions/mean_terminated_length": 710.8978271484375, + "completions/min_length": 145.0, + "completions/min_terminated_length": 145.0, + "epoch": 0.8712127677733208, + "grad_norm": 2.072315216064453, + "kl": 5.8515625, + "learning_rate": 1.4486355305349583e-07, + "loss": 0.3489, + "num_tokens": 1365130576.0, + "reward": 1.9111328125, + "reward_std": 0.4487929940223694, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.8984375, + "rewards/format_reward/std": 0.30236753821372986, + "rewards/tag_count_reward/mean": 0.9423828125, + "rewards/tag_count_reward/std": 0.1833334118127823, + "step": 2552 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2007.0, + "completions/mean_length": 826.181640625, + "completions/mean_terminated_length": 791.8333129882812, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "epoch": 0.8715541520867116, + "grad_norm": 1.1610126495361328, + "kl": 3.9609375, + "learning_rate": 1.4463049250916792e-07, + "loss": 0.2513, + "num_tokens": 1365631373.0, + "reward": 1.91015625, + "reward_std": 0.35444962978363037, + "rewards/accuracy_reward/mean": 0.033203125, + "rewards/accuracy_reward/std": 0.17934183776378632, + "rewards/format_reward/mean": 0.916015625, + "rewards/format_reward/std": 0.2776356339454651, + "rewards/tag_count_reward/mean": 0.9609375, + "rewards/tag_count_reward/std": 0.1396721750497818, + "step": 2553 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1865.0, + "completions/mean_length": 764.099609375, + "completions/mean_terminated_length": 730.6513061523438, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "epoch": 0.8718955364001024, + "grad_norm": 1.0609973669052124, + "kl": 4.712890625, + "learning_rate": 1.4439800731271267e-07, + "loss": 0.3242, + "num_tokens": 1366106272.0, + "reward": 1.92333984375, + "reward_std": 0.3831869959831238, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.904296875, + "rewards/format_reward/std": 0.2944713830947876, + "rewards/tag_count_reward/mean": 0.95654296875, + "rewards/tag_count_reward/std": 0.1490057110786438, + "step": 2554 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2015.0, + "completions/mean_length": 916.3125, + "completions/mean_terminated_length": 886.8296508789062, + "completions/min_length": 188.0, + "completions/min_terminated_length": 188.0, + "epoch": 0.8722369207134932, + "grad_norm": 1.3187130689620972, + "kl": 4.90234375, + "learning_rate": 1.4416609779410049e-07, + "loss": 0.3169, + "num_tokens": 1366657712.0, + "reward": 1.84130859375, + "reward_std": 0.4678717851638794, + "rewards/accuracy_reward/mean": 0.025390625, + "rewards/accuracy_reward/std": 0.15746226906776428, + "rewards/format_reward/mean": 0.8828125, + "rewards/format_reward/std": 0.32195815443992615, + "rewards/tag_count_reward/mean": 0.93310546875, + "rewards/tag_count_reward/std": 0.19038206338882446, + "step": 2555 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1995.0, + "completions/mean_length": 765.8828125, + "completions/mean_terminated_length": 745.5317993164062, + "completions/min_length": 158.0, + "completions/min_terminated_length": 158.0, + "epoch": 0.872578305026884, + "grad_norm": 1.5510096549987793, + "kl": 4.05859375, + "learning_rate": 1.4393476428248442e-07, + "loss": 0.2521, + "num_tokens": 1367136164.0, + "reward": 1.9287109375, + "reward_std": 0.4307078719139099, + "rewards/accuracy_reward/mean": 0.07661290466785431, + "rewards/accuracy_reward/std": 0.2662447690963745, + "rewards/format_reward/mean": 0.91015625, + "rewards/format_reward/std": 0.2862374484539032, + "rewards/tag_count_reward/mean": 0.9443359375, + "rewards/tag_count_reward/std": 0.17226234078407288, + "step": 2556 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1971.0, + "completions/mean_length": 786.2734375, + "completions/mean_terminated_length": 753.40283203125, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "epoch": 0.8729196893402749, + "grad_norm": 2.504319906234741, + "kl": 4.68359375, + "learning_rate": 1.4370400710620017e-07, + "loss": 0.3464, + "num_tokens": 1367612320.0, + "reward": 1.9033203125, + "reward_std": 0.45469242334365845, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.900390625, + "rewards/format_reward/std": 0.29977133870124817, + "rewards/tag_count_reward/mean": 0.9482421875, + "rewards/tag_count_reward/std": 0.1625591367483139, + "step": 2557 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1991.0, + "completions/mean_length": 785.9296875, + "completions/mean_terminated_length": 747.8389892578125, + "completions/min_length": 24.0, + "completions/min_terminated_length": 24.0, + "epoch": 0.8732610736536656, + "grad_norm": 1.2066786289215088, + "kl": 5.4296875, + "learning_rate": 1.4347382659276529e-07, + "loss": 0.3174, + "num_tokens": 1368091916.0, + "reward": 1.86083984375, + "reward_std": 0.4825897514820099, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.876953125, + "rewards/format_reward/std": 0.32881227135658264, + "rewards/tag_count_reward/mean": 0.93115234375, + "rewards/tag_count_reward/std": 0.18773873150348663, + "step": 2558 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1996.0, + "completions/mean_length": 869.80078125, + "completions/mean_terminated_length": 836.6787109375, + "completions/min_length": 62.0, + "completions/min_terminated_length": 62.0, + "epoch": 0.8736024579670564, + "grad_norm": 1.2075902223587036, + "kl": 5.69140625, + "learning_rate": 1.4324422306887873e-07, + "loss": 0.3744, + "num_tokens": 1368614374.0, + "reward": 1.87451171875, + "reward_std": 0.5310301780700684, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.87890625, + "rewards/format_reward/std": 0.3265552520751953, + "rewards/tag_count_reward/mean": 0.93310546875, + "rewards/tag_count_reward/std": 0.19419844448566437, + "step": 2559 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2015.0, + "completions/mean_length": 796.73046875, + "completions/mean_terminated_length": 758.9657592773438, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "epoch": 0.8739438422804472, + "grad_norm": 0.9230890870094299, + "kl": 5.16796875, + "learning_rate": 1.430151968604211e-07, + "loss": 0.3276, + "num_tokens": 1369103500.0, + "reward": 1.9501953125, + "reward_std": 0.5184177160263062, + "rewards/accuracy_reward/mean": 0.123046875, + "rewards/accuracy_reward/std": 0.32881227135658264, + "rewards/format_reward/mean": 0.888671875, + "rewards/format_reward/std": 0.31484565138816833, + "rewards/tag_count_reward/mean": 0.9384765625, + "rewards/tag_count_reward/std": 0.17729215323925018, + "step": 2560 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1959.0, + "completions/mean_length": 785.23046875, + "completions/mean_terminated_length": 760.0757446289062, + "completions/min_length": 155.0, + "completions/min_terminated_length": 155.0, + "epoch": 0.874285226593838, + "grad_norm": 1.4074889421463013, + "kl": 3.5390625, + "learning_rate": 1.4278674829245282e-07, + "loss": 0.2361, + "num_tokens": 1369577762.0, + "reward": 1.9853515625, + "reward_std": 0.3713374137878418, + "rewards/accuracy_reward/mean": 0.099609375, + "rewards/accuracy_reward/std": 0.29977133870124817, + "rewards/format_reward/mean": 0.927734375, + "rewards/format_reward/std": 0.2591804563999176, + "rewards/tag_count_reward/mean": 0.9580078125, + "rewards/tag_count_reward/std": 0.1522638499736786, + "step": 2561 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1910.0, + "completions/mean_length": 839.52734375, + "completions/mean_terminated_length": 787.8411865234375, + "completions/min_length": 59.0, + "completions/min_terminated_length": 59.0, + "epoch": 0.8746266109072288, + "grad_norm": 3.308898448944092, + "kl": 7.6796875, + "learning_rate": 1.425588776892151e-07, + "loss": 0.4635, + "num_tokens": 1370086176.0, + "reward": 1.900390625, + "reward_std": 0.5681383609771729, + "rewards/accuracy_reward/mean": 0.109375, + "rewards/accuracy_reward/std": 0.31241437792778015, + "rewards/format_reward/mean": 0.8671875, + "rewards/format_reward/std": 0.33970388770103455, + "rewards/tag_count_reward/mean": 0.923828125, + "rewards/tag_count_reward/std": 0.20466333627700806, + "step": 2562 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1964.0, + "completions/mean_length": 724.361328125, + "completions/mean_terminated_length": 687.1505737304688, + "completions/min_length": 110.0, + "completions/min_terminated_length": 110.0, + "epoch": 0.8749679952206196, + "grad_norm": 1.095280647277832, + "kl": 5.5, + "learning_rate": 1.423315853741285e-07, + "loss": 0.3733, + "num_tokens": 1370535753.0, + "reward": 1.94873046875, + "reward_std": 0.4907839596271515, + "rewards/accuracy_reward/mean": 0.10080645233392715, + "rewards/accuracy_reward/std": 0.30137622356414795, + "rewards/format_reward/mean": 0.900390625, + "rewards/format_reward/std": 0.29977133870124817, + "rewards/tag_count_reward/mean": 0.95068359375, + "rewards/tag_count_reward/std": 0.15525536239147186, + "step": 2563 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2004.0, + "completions/mean_length": 799.357421875, + "completions/mean_terminated_length": 761.6719970703125, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "epoch": 0.8753093795340104, + "grad_norm": 3.3422834873199463, + "kl": 7.0546875, + "learning_rate": 1.4210487166979283e-07, + "loss": 0.4382, + "num_tokens": 1371035856.0, + "reward": 1.81787109375, + "reward_std": 0.534123957157135, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.849609375, + "rewards/format_reward/std": 0.35780346393585205, + "rewards/tag_count_reward/mean": 0.92724609375, + "rewards/tag_count_reward/std": 0.19397689402103424, + "step": 2564 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2020.0, + "completions/mean_length": 823.25, + "completions/mean_terminated_length": 776.0486450195312, + "completions/min_length": 46.0, + "completions/min_terminated_length": 46.0, + "epoch": 0.8756507638474013, + "grad_norm": 3.232412338256836, + "kl": 6.9140625, + "learning_rate": 1.4187873689798684e-07, + "loss": 0.4274, + "num_tokens": 1371533728.0, + "reward": 1.83642578125, + "reward_std": 0.5565227270126343, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.92236328125, + "rewards/tag_count_reward/std": 0.1964779943227768, + "step": 2565 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1976.0, + "completions/mean_length": 809.40625, + "completions/mean_terminated_length": 764.2753295898438, + "completions/min_length": 84.0, + "completions/min_terminated_length": 84.0, + "epoch": 0.875992148160792, + "grad_norm": 2.557537794113159, + "kl": 6.6015625, + "learning_rate": 1.416531813796674e-07, + "loss": 0.4312, + "num_tokens": 1372028720.0, + "reward": 1.80859375, + "reward_std": 0.5407989621162415, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.919921875, + "rewards/tag_count_reward/std": 0.19705164432525635, + "step": 2566 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2024.0, + "completions/mean_length": 825.794921875, + "completions/mean_terminated_length": 783.8202514648438, + "completions/min_length": 96.0, + "completions/min_terminated_length": 96.0, + "epoch": 0.8763335324741828, + "grad_norm": 1.5472102165222168, + "kl": 5.2734375, + "learning_rate": 1.4142820543496936e-07, + "loss": 0.322, + "num_tokens": 1372530407.0, + "reward": 1.9423828125, + "reward_std": 0.5201644897460938, + "rewards/accuracy_reward/mean": 0.1328125, + "rewards/accuracy_reward/std": 0.33970388770103455, + "rewards/format_reward/mean": 0.875, + "rewards/format_reward/std": 0.3310423493385315, + "rewards/tag_count_reward/mean": 0.9345703125, + "rewards/tag_count_reward/std": 0.1847042590379715, + "step": 2567 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1862.0, + "completions/mean_length": 820.751953125, + "completions/mean_terminated_length": 793.806396484375, + "completions/min_length": 160.0, + "completions/min_terminated_length": 160.0, + "epoch": 0.8766749167875736, + "grad_norm": 1.468971848487854, + "kl": 4.44921875, + "learning_rate": 1.4120380938320487e-07, + "loss": 0.3006, + "num_tokens": 1373023064.0, + "reward": 1.86376953125, + "reward_std": 0.4445911645889282, + "rewards/accuracy_reward/mean": 0.021484375, + "rewards/accuracy_reward/std": 0.14513419568538666, + "rewards/format_reward/mean": 0.896484375, + "rewards/format_reward/std": 0.30492907762527466, + "rewards/tag_count_reward/mean": 0.94580078125, + "rewards/tag_count_reward/std": 0.1687902808189392, + "step": 2568 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1925.0, + "completions/mean_length": 778.3359375, + "completions/mean_terminated_length": 747.864013671875, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "epoch": 0.8770163011009644, + "grad_norm": 2.1014490127563477, + "kl": 4.40625, + "learning_rate": 1.4097999354286316e-07, + "loss": 0.2878, + "num_tokens": 1373492564.0, + "reward": 1.86767578125, + "reward_std": 0.45762932300567627, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.876953125, + "rewards/format_reward/std": 0.32881227135658264, + "rewards/tag_count_reward/mean": 0.94189453125, + "rewards/tag_count_reward/std": 0.17180897295475006, + "step": 2569 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1937.0, + "completions/mean_length": 850.48828125, + "completions/mean_terminated_length": 806.854248046875, + "completions/min_length": 71.0, + "completions/min_terminated_length": 71.0, + "epoch": 0.8773576854143552, + "grad_norm": 1.2898650169372559, + "kl": 5.234375, + "learning_rate": 1.4075675823160982e-07, + "loss": 0.3387, + "num_tokens": 1374011678.0, + "reward": 1.89208984375, + "reward_std": 0.43215885758399963, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.896484375, + "rewards/format_reward/std": 0.30492907762527466, + "rewards/tag_count_reward/mean": 0.94677734375, + "rewards/tag_count_reward/std": 0.16691738367080688, + "step": 2570 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1970.0, + "completions/mean_length": 807.484375, + "completions/mean_terminated_length": 772.6104125976562, + "completions/min_length": 164.0, + "completions/min_terminated_length": 164.0, + "epoch": 0.877699069727746, + "grad_norm": 2.6725640296936035, + "kl": 5.8046875, + "learning_rate": 1.4053410376628647e-07, + "loss": 0.4056, + "num_tokens": 1374494390.0, + "reward": 1.875, + "reward_std": 0.4745900630950928, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.87890625, + "rewards/format_reward/std": 0.3265552520751953, + "rewards/tag_count_reward/mean": 0.94140625, + "rewards/tag_count_reward/std": 0.17411890625953674, + "step": 2571 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1892.0, + "completions/mean_length": 822.611328125, + "completions/mean_terminated_length": 788.16259765625, + "completions/min_length": 188.0, + "completions/min_terminated_length": 188.0, + "epoch": 0.8780404540411368, + "grad_norm": 3.3297979831695557, + "kl": 5.08203125, + "learning_rate": 1.403120304629106e-07, + "loss": 0.3298, + "num_tokens": 1374999887.0, + "reward": 1.86962890625, + "reward_std": 0.5119246244430542, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.8828125, + "rewards/format_reward/std": 0.32195815443992615, + "rewards/tag_count_reward/mean": 0.93798828125, + "rewards/tag_count_reward/std": 0.17883963882923126, + "step": 2572 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1806.0, + "completions/mean_length": 835.00390625, + "completions/mean_terminated_length": 798.3943481445312, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "epoch": 0.8783818383545277, + "grad_norm": 0.9163107872009277, + "kl": 5.84375, + "learning_rate": 1.4009053863667448e-07, + "loss": 0.343, + "num_tokens": 1375504465.0, + "reward": 1.89013671875, + "reward_std": 0.5146956443786621, + "rewards/accuracy_reward/mean": 0.08467742055654526, + "rewards/accuracy_reward/std": 0.278682142496109, + "rewards/format_reward/mean": 0.875, + "rewards/format_reward/std": 0.3310423493385315, + "rewards/tag_count_reward/mean": 0.93310546875, + "rewards/tag_count_reward/std": 0.18844488263130188, + "step": 2573 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 833.171875, + "completions/mean_terminated_length": 801.5230712890625, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "epoch": 0.8787232226679184, + "grad_norm": 1.5819618701934814, + "kl": 7.4765625, + "learning_rate": 1.3986962860194528e-07, + "loss": 0.4639, + "num_tokens": 1376011033.0, + "reward": 1.865234375, + "reward_std": 0.6026434898376465, + "rewards/accuracy_reward/mean": 0.1015625, + "rewards/accuracy_reward/std": 0.30236753821372986, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.916015625, + "rewards/tag_count_reward/std": 0.2087314873933792, + "step": 2574 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 839.353515625, + "completions/mean_terminated_length": 810.3460083007812, + "completions/min_length": 121.0, + "completions/min_terminated_length": 121.0, + "epoch": 0.8790646069813092, + "grad_norm": 1.3439744710922241, + "kl": 6.078125, + "learning_rate": 1.396493006722645e-07, + "loss": 0.3989, + "num_tokens": 1376518014.0, + "reward": 1.90966796875, + "reward_std": 0.4882264733314514, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.88671875, + "rewards/format_reward/std": 0.3172462284564972, + "rewards/tag_count_reward/mean": 0.93896484375, + "rewards/tag_count_reward/std": 0.1729232519865036, + "step": 2575 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1999.0, + "completions/mean_length": 840.857421875, + "completions/mean_terminated_length": 784.0797119140625, + "completions/min_length": 185.0, + "completions/min_terminated_length": 185.0, + "epoch": 0.8794059912947, + "grad_norm": 1.4315595626831055, + "kl": 7.328125, + "learning_rate": 1.3942955516034715e-07, + "loss": 0.4431, + "num_tokens": 1377022229.0, + "reward": 1.8798828125, + "reward_std": 0.5710894465446472, + "rewards/accuracy_reward/mean": 0.125, + "rewards/accuracy_reward/std": 0.3310528099536896, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.9130859375, + "rewards/tag_count_reward/std": 0.21561963856220245, + "step": 2576 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 857.81640625, + "completions/mean_terminated_length": 814.4494018554688, + "completions/min_length": 127.0, + "completions/min_terminated_length": 127.0, + "epoch": 0.8797473756080908, + "grad_norm": 1.4523320198059082, + "kl": 6.8515625, + "learning_rate": 1.3921039237808198e-07, + "loss": 0.4524, + "num_tokens": 1377537447.0, + "reward": 1.86669921875, + "reward_std": 0.5333712100982666, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.8671875, + "rewards/format_reward/std": 0.33970388770103455, + "rewards/tag_count_reward/mean": 0.93115234375, + "rewards/tag_count_reward/std": 0.18445254862308502, + "step": 2577 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 812.955078125, + "completions/mean_terminated_length": 778.2349243164062, + "completions/min_length": 52.0, + "completions/min_terminated_length": 52.0, + "epoch": 0.8800887599214816, + "grad_norm": 2.0251107215881348, + "kl": 5.703125, + "learning_rate": 1.3899181263653026e-07, + "loss": 0.3532, + "num_tokens": 1378044208.0, + "reward": 1.8525390625, + "reward_std": 0.5274684429168701, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.86328125, + "rewards/format_reward/std": 0.3438861668109894, + "rewards/tag_count_reward/mean": 0.9306640625, + "rewards/tag_count_reward/std": 0.19175048172473907, + "step": 2578 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1912.0, + "completions/mean_length": 843.6953125, + "completions/mean_terminated_length": 794.7398071289062, + "completions/min_length": 125.0, + "completions/min_terminated_length": 125.0, + "epoch": 0.8804301442348724, + "grad_norm": 1.2781013250350952, + "kl": 8.453125, + "learning_rate": 1.3877381624592616e-07, + "loss": 0.5084, + "num_tokens": 1378553124.0, + "reward": 1.84619140625, + "reward_std": 0.6146891117095947, + "rewards/accuracy_reward/mean": 0.087890625, + "rewards/accuracy_reward/std": 0.2834126651287079, + "rewards/format_reward/mean": 0.841796875, + "rewards/format_reward/std": 0.36528825759887695, + "rewards/tag_count_reward/mean": 0.91650390625, + "rewards/tag_count_reward/std": 0.20686857402324677, + "step": 2579 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1996.0, + "completions/mean_length": 826.57421875, + "completions/mean_terminated_length": 779.5009765625, + "completions/min_length": 170.0, + "completions/min_terminated_length": 170.0, + "epoch": 0.8807715285482632, + "grad_norm": 2.706282138824463, + "kl": 7.61328125, + "learning_rate": 1.3855640351567553e-07, + "loss": 0.4517, + "num_tokens": 1379047098.0, + "reward": 1.9033203125, + "reward_std": 0.614270031452179, + "rewards/accuracy_reward/mean": 0.14453125, + "rewards/accuracy_reward/std": 0.35197147727012634, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.9208984375, + "rewards/tag_count_reward/std": 0.19744645059108734, + "step": 2580 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1996.0, + "completions/mean_length": 825.232421875, + "completions/mean_terminated_length": 785.7882690429688, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.881112912861654, + "grad_norm": 1.6486424207687378, + "kl": 8.2578125, + "learning_rate": 1.3833957475435613e-07, + "loss": 0.4868, + "num_tokens": 1379555665.0, + "reward": 1.755859375, + "reward_std": 0.5890187621116638, + "rewards/accuracy_reward/mean": 0.04032257944345474, + "rewards/accuracy_reward/std": 0.19691328704357147, + "rewards/format_reward/mean": 0.810546875, + "rewards/format_reward/std": 0.3922513723373413, + "rewards/tag_count_reward/mean": 0.90625, + "rewards/tag_count_reward/std": 0.2144487351179123, + "step": 2581 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2010.0, + "completions/mean_length": 864.81640625, + "completions/mean_terminated_length": 801.5184936523438, + "completions/min_length": 139.0, + "completions/min_terminated_length": 139.0, + "epoch": 0.8814542971750448, + "grad_norm": 2.1603972911834717, + "kl": 8.6171875, + "learning_rate": 1.3812333026971663e-07, + "loss": 0.527, + "num_tokens": 1380081219.0, + "reward": 1.74462890625, + "reward_std": 0.5655951499938965, + "rewards/accuracy_reward/mean": 0.02734375, + "rewards/accuracy_reward/std": 0.16324250400066376, + "rewards/format_reward/mean": 0.810546875, + "rewards/format_reward/std": 0.3922513723373413, + "rewards/tag_count_reward/mean": 0.90673828125, + "rewards/tag_count_reward/std": 0.21323274075984955, + "step": 2582 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1960.0, + "completions/mean_length": 871.83203125, + "completions/mean_terminated_length": 811.453857421875, + "completions/min_length": 168.0, + "completions/min_terminated_length": 168.0, + "epoch": 0.8817956814884356, + "grad_norm": 1.896267294883728, + "kl": 7.8515625, + "learning_rate": 1.3790767036867645e-07, + "loss": 0.4951, + "num_tokens": 1380610269.0, + "reward": 1.79296875, + "reward_std": 0.6150627136230469, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.814453125, + "rewards/format_reward/std": 0.38912075757980347, + "rewards/tag_count_reward/mean": 0.90234375, + "rewards/tag_count_reward/std": 0.22225576639175415, + "step": 2583 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 857.380859375, + "completions/mean_terminated_length": 823.9096069335938, + "completions/min_length": 174.0, + "completions/min_terminated_length": 174.0, + "epoch": 0.8821370658018264, + "grad_norm": 1.679373860359192, + "kl": 6.578125, + "learning_rate": 1.3769259535732561e-07, + "loss": 0.4107, + "num_tokens": 1381125728.0, + "reward": 1.89794921875, + "reward_std": 0.6069402694702148, + "rewards/accuracy_reward/mean": 0.130859375, + "rewards/accuracy_reward/std": 0.33757632970809937, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.92333984375, + "rewards/tag_count_reward/std": 0.1981005072593689, + "step": 2584 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1908.0, + "completions/mean_length": 882.2265625, + "completions/mean_terminated_length": 824.8933715820312, + "completions/min_length": 110.0, + "completions/min_terminated_length": 110.0, + "epoch": 0.8824784501152172, + "grad_norm": 2.3373751640319824, + "kl": 6.546875, + "learning_rate": 1.374781055409235e-07, + "loss": 0.4046, + "num_tokens": 1381648676.0, + "reward": 1.85546875, + "reward_std": 0.5630089640617371, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.91796875, + "rewards/tag_count_reward/std": 0.20833741128444672, + "step": 2585 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.056640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1974.0, + "completions/mean_length": 834.037109375, + "completions/mean_terminated_length": 761.1491088867188, + "completions/min_length": 62.0, + "completions/min_terminated_length": 62.0, + "epoch": 0.882819834428608, + "grad_norm": 1.7196927070617676, + "kl": 6.6640625, + "learning_rate": 1.372642012238993e-07, + "loss": 0.4339, + "num_tokens": 1382149511.0, + "reward": 1.8330078125, + "reward_std": 0.5458844900131226, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.9169921875, + "rewards/tag_count_reward/std": 0.2043900191783905, + "step": 2586 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1985.0, + "completions/mean_length": 851.55859375, + "completions/mean_terminated_length": 802.9227294921875, + "completions/min_length": 118.0, + "completions/min_terminated_length": 118.0, + "epoch": 0.8831612187419988, + "grad_norm": 2.913691759109497, + "kl": 6.65625, + "learning_rate": 1.3705088270985103e-07, + "loss": 0.457, + "num_tokens": 1382662069.0, + "reward": 1.82958984375, + "reward_std": 0.5881297588348389, + "rewards/accuracy_reward/mean": 0.08467742055654526, + "rewards/accuracy_reward/std": 0.278682142496109, + "rewards/format_reward/mean": 0.833984375, + "rewards/format_reward/std": 0.3724585771560669, + "rewards/tag_count_reward/mean": 0.91357421875, + "rewards/tag_count_reward/std": 0.21267634630203247, + "step": 2587 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1998.0, + "completions/mean_length": 881.123046875, + "completions/mean_terminated_length": 816.1629028320312, + "completions/min_length": 6.0, + "completions/min_terminated_length": 6.0, + "epoch": 0.8835026030553896, + "grad_norm": 2.5694093704223633, + "kl": 6.5703125, + "learning_rate": 1.3683815030154538e-07, + "loss": 0.4404, + "num_tokens": 1383187316.0, + "reward": 1.7939453125, + "reward_std": 0.6087914705276489, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.826171875, + "rewards/format_reward/std": 0.3793322443962097, + "rewards/tag_count_reward/mean": 0.9130859375, + "rewards/tag_count_reward/std": 0.2081148475408554, + "step": 2588 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 878.44140625, + "completions/mean_terminated_length": 810.7809448242188, + "completions/min_length": 113.0, + "completions/min_terminated_length": 113.0, + "epoch": 0.8838439873687804, + "grad_norm": 2.169834613800049, + "kl": 6.734375, + "learning_rate": 1.3662600430091707e-07, + "loss": 0.4543, + "num_tokens": 1383720246.0, + "reward": 1.78662109375, + "reward_std": 0.5730187892913818, + "rewards/accuracy_reward/mean": 0.0463709682226181, + "rewards/accuracy_reward/std": 0.21049949526786804, + "rewards/format_reward/mean": 0.828125, + "rewards/format_reward/std": 0.3776407241821289, + "rewards/tag_count_reward/mean": 0.91357421875, + "rewards/tag_count_reward/std": 0.21152304112911224, + "step": 2589 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2041.0, + "completions/mean_length": 864.8125, + "completions/mean_terminated_length": 819.2129516601562, + "completions/min_length": 172.0, + "completions/min_terminated_length": 172.0, + "epoch": 0.8841853716821712, + "grad_norm": 1.9821420907974243, + "kl": 6.7578125, + "learning_rate": 1.3641444500906846e-07, + "loss": 0.4474, + "num_tokens": 1384244406.0, + "reward": 1.810546875, + "reward_std": 0.5634486675262451, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.9140625, + "rewards/tag_count_reward/std": 0.21601147949695587, + "step": 2590 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2021.0, + "completions/mean_length": 852.43359375, + "completions/mean_terminated_length": 796.2003784179688, + "completions/min_length": 123.0, + "completions/min_terminated_length": 123.0, + "epoch": 0.884526755995562, + "grad_norm": 1.7044569253921509, + "kl": 7.09375, + "learning_rate": 1.3620347272626933e-07, + "loss": 0.447, + "num_tokens": 1384750324.0, + "reward": 1.857421875, + "reward_std": 0.5381364822387695, + "rewards/accuracy_reward/mean": 0.087890625, + "rewards/accuracy_reward/std": 0.2834126651287079, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.91796875, + "rewards/tag_count_reward/std": 0.20478467643260956, + "step": 2591 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1850.0, + "completions/mean_length": 818.771484375, + "completions/mean_terminated_length": 760.9550170898438, + "completions/min_length": 96.0, + "completions/min_terminated_length": 96.0, + "epoch": 0.8848681403089528, + "grad_norm": 1.2714744806289673, + "kl": 7.65625, + "learning_rate": 1.359930877519562e-07, + "loss": 0.4809, + "num_tokens": 1385238815.0, + "reward": 1.84912109375, + "reward_std": 0.5803292989730835, + "rewards/accuracy_reward/mean": 0.087890625, + "rewards/accuracy_reward/std": 0.2834126651287079, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.92138671875, + "rewards/tag_count_reward/std": 0.19979596138000488, + "step": 2592 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1976.0, + "completions/mean_length": 842.171875, + "completions/mean_terminated_length": 777.6625366210938, + "completions/min_length": 88.0, + "completions/min_terminated_length": 88.0, + "epoch": 0.8852095246223436, + "grad_norm": 1.324734091758728, + "kl": 5.6796875, + "learning_rate": 1.3578329038473222e-07, + "loss": 0.3821, + "num_tokens": 1385745831.0, + "reward": 1.8408203125, + "reward_std": 0.49896273016929626, + "rewards/accuracy_reward/mean": 0.05645161122083664, + "rewards/accuracy_reward/std": 0.23102475702762604, + "rewards/format_reward/mean": 0.859375, + "rewards/format_reward/std": 0.3479743003845215, + "rewards/tag_count_reward/mean": 0.9267578125, + "rewards/tag_count_reward/std": 0.1972333937883377, + "step": 2593 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2010.0, + "completions/mean_length": 788.927734375, + "completions/mean_terminated_length": 756.1262817382812, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "epoch": 0.8855509089357344, + "grad_norm": 1.3588322401046753, + "kl": 7.2421875, + "learning_rate": 1.355740809223662e-07, + "loss": 0.4772, + "num_tokens": 1386223330.0, + "reward": 1.92333984375, + "reward_std": 0.5310149788856506, + "rewards/accuracy_reward/mean": 0.1328125, + "rewards/accuracy_reward/std": 0.33970388770103455, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.92919921875, + "rewards/tag_count_reward/std": 0.19089330732822418, + "step": 2594 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 852.869140625, + "completions/mean_terminated_length": 783.7293090820312, + "completions/min_length": 225.0, + "completions/min_terminated_length": 225.0, + "epoch": 0.8858922932491252, + "grad_norm": 1.2075051069259644, + "kl": 6.9765625, + "learning_rate": 1.3536545966179274e-07, + "loss": 0.4607, + "num_tokens": 1386729151.0, + "reward": 1.85302734375, + "reward_std": 0.4970835745334625, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.92919921875, + "rewards/tag_count_reward/std": 0.19595204293727875, + "step": 2595 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2022.0, + "completions/mean_length": 890.853515625, + "completions/mean_terminated_length": 836.4273681640625, + "completions/min_length": 129.0, + "completions/min_terminated_length": 129.0, + "epoch": 0.886233677562516, + "grad_norm": 0.8362195491790771, + "kl": 6.6953125, + "learning_rate": 1.3515742689911166e-07, + "loss": 0.4196, + "num_tokens": 1387257892.0, + "reward": 1.8447265625, + "reward_std": 0.5106940269470215, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.9267578125, + "rewards/tag_count_reward/std": 0.19908507168293, + "step": 2596 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 763.521484375, + "completions/mean_terminated_length": 730.05810546875, + "completions/min_length": 96.0, + "completions/min_terminated_length": 96.0, + "epoch": 0.8865750618759068, + "grad_norm": 0.9312205910682678, + "kl": 4.65625, + "learning_rate": 1.3494998292958725e-07, + "loss": 0.2835, + "num_tokens": 1387725487.0, + "reward": 1.89501953125, + "reward_std": 0.4495265483856201, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.890625, + "rewards/format_reward/std": 0.31241437792778015, + "rewards/tag_count_reward/mean": 0.94384765625, + "rewards/tag_count_reward/std": 0.17387108504772186, + "step": 2597 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1930.0, + "completions/mean_length": 846.666015625, + "completions/mean_terminated_length": 810.408447265625, + "completions/min_length": 170.0, + "completions/min_terminated_length": 170.0, + "epoch": 0.8869164461892975, + "grad_norm": 1.2470439672470093, + "kl": 7.0703125, + "learning_rate": 1.3474312804764853e-07, + "loss": 0.4707, + "num_tokens": 1388233700.0, + "reward": 1.84033203125, + "reward_std": 0.5441080331802368, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.86328125, + "rewards/format_reward/std": 0.3438861668109894, + "rewards/tag_count_reward/mean": 0.93017578125, + "rewards/tag_count_reward/std": 0.19630283117294312, + "step": 2598 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2014.0, + "completions/mean_length": 819.166015625, + "completions/mean_terminated_length": 763.9938354492188, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "epoch": 0.8872578305026884, + "grad_norm": 1.1735024452209473, + "kl": 6.5625, + "learning_rate": 1.345368625468879e-07, + "loss": 0.4053, + "num_tokens": 1388731097.0, + "reward": 1.888671875, + "reward_std": 0.499426007270813, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.875, + "rewards/format_reward/std": 0.3310423493385315, + "rewards/tag_count_reward/mean": 0.939453125, + "rewards/tag_count_reward/std": 0.1810387820005417, + "step": 2599 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1895.0, + "completions/mean_length": 827.431640625, + "completions/mean_terminated_length": 788.0584716796875, + "completions/min_length": 135.0, + "completions/min_terminated_length": 135.0, + "epoch": 0.8875992148160792, + "grad_norm": 0.7364870309829712, + "kl": 7.4296875, + "learning_rate": 1.3433118672006173e-07, + "loss": 0.4804, + "num_tokens": 1389228198.0, + "reward": 1.89599609375, + "reward_std": 0.5626035928726196, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.93310546875, + "rewards/tag_count_reward/std": 0.1838454306125641, + "step": 2600 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1977.0, + "completions/mean_length": 824.7578125, + "completions/mean_terminated_length": 797.9002075195312, + "completions/min_length": 71.0, + "completions/min_terminated_length": 71.0, + "epoch": 0.88794059912947, + "grad_norm": 0.8736045360565186, + "kl": 5.765625, + "learning_rate": 1.3412610085908912e-07, + "loss": 0.3384, + "num_tokens": 1389723914.0, + "reward": 1.86865234375, + "reward_std": 0.4505782723426819, + "rewards/accuracy_reward/mean": 0.03515625, + "rewards/accuracy_reward/std": 0.1843547374010086, + "rewards/format_reward/mean": 0.88671875, + "rewards/format_reward/std": 0.3172462284564972, + "rewards/tag_count_reward/mean": 0.94677734375, + "rewards/tag_count_reward/std": 0.16837652027606964, + "step": 2601 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1981.0, + "completions/mean_length": 804.08203125, + "completions/mean_terminated_length": 766.5391845703125, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "epoch": 0.8882819834428608, + "grad_norm": 1.735905647277832, + "kl": 6.46875, + "learning_rate": 1.3392160525505191e-07, + "loss": 0.4185, + "num_tokens": 1390210644.0, + "reward": 1.97119140625, + "reward_std": 0.5567293167114258, + "rewards/accuracy_reward/mean": 0.13709677755832672, + "rewards/accuracy_reward/std": 0.34429675340652466, + "rewards/format_reward/mean": 0.896484375, + "rewards/format_reward/std": 0.30492907762527466, + "rewards/tag_count_reward/mean": 0.94189453125, + "rewards/tag_count_reward/std": 0.1794690042734146, + "step": 2602 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1984.0, + "completions/mean_length": 764.5546875, + "completions/mean_terminated_length": 738.9880981445312, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "epoch": 0.8886233677562516, + "grad_norm": 1.4352848529815674, + "kl": 6.046875, + "learning_rate": 1.3371770019819433e-07, + "loss": 0.3766, + "num_tokens": 1390676720.0, + "reward": 1.90673828125, + "reward_std": 0.46058928966522217, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.890625, + "rewards/format_reward/std": 0.31241437792778015, + "rewards/tag_count_reward/mean": 0.94189453125, + "rewards/tag_count_reward/std": 0.18217463791370392, + "step": 2603 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1789.0, + "completions/mean_length": 777.220703125, + "completions/mean_terminated_length": 744.1142578125, + "completions/min_length": 179.0, + "completions/min_terminated_length": 179.0, + "epoch": 0.8889647520696424, + "grad_norm": 0.971156120300293, + "kl": 6.19140625, + "learning_rate": 1.3351438597792218e-07, + "loss": 0.3897, + "num_tokens": 1391146785.0, + "reward": 1.93408203125, + "reward_std": 0.47433918714523315, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.904296875, + "rewards/format_reward/std": 0.2944713830947876, + "rewards/tag_count_reward/mean": 0.93994140625, + "rewards/tag_count_reward/std": 0.18880455195903778, + "step": 2604 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1734.0, + "completions/mean_length": 726.63671875, + "completions/mean_terminated_length": 708.32080078125, + "completions/min_length": 149.0, + "completions/min_terminated_length": 149.0, + "epoch": 0.8893061363830332, + "grad_norm": 0.963898777961731, + "kl": 4.296875, + "learning_rate": 1.3331166288280295e-07, + "loss": 0.2596, + "num_tokens": 1391593767.0, + "reward": 1.9560546875, + "reward_std": 0.44860565662384033, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.9140625, + "rewards/format_reward/std": 0.28054583072662354, + "rewards/tag_count_reward/mean": 0.9580078125, + "rewards/tag_count_reward/std": 0.1448541134595871, + "step": 2605 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 794.787109375, + "completions/mean_terminated_length": 767.271484375, + "completions/min_length": 5.0, + "completions/min_terminated_length": 5.0, + "epoch": 0.889647520696424, + "grad_norm": 1.4902775287628174, + "kl": 5.7890625, + "learning_rate": 1.3310953120056488e-07, + "loss": 0.3435, + "num_tokens": 1392072906.0, + "reward": 2.00146484375, + "reward_std": 0.5466192364692688, + "rewards/accuracy_reward/mean": 0.166015625, + "rewards/accuracy_reward/std": 0.3724585771560669, + "rewards/format_reward/mean": 0.892578125, + "rewards/format_reward/std": 0.30995169281959534, + "rewards/tag_count_reward/mean": 0.94287109375, + "rewards/tag_count_reward/std": 0.17841704189777374, + "step": 2606 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1941.0, + "completions/mean_length": 809.595703125, + "completions/mean_terminated_length": 761.8681640625, + "completions/min_length": 102.0, + "completions/min_terminated_length": 102.0, + "epoch": 0.8899889050098148, + "grad_norm": 1.5362666845321655, + "kl": 7.359375, + "learning_rate": 1.3290799121809702e-07, + "loss": 0.4408, + "num_tokens": 1392561467.0, + "reward": 1.8310546875, + "reward_std": 0.5468693375587463, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.9267578125, + "rewards/tag_count_reward/std": 0.1959892213344574, + "step": 2607 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 873.70703125, + "completions/mean_terminated_length": 835.8265991210938, + "completions/min_length": 158.0, + "completions/min_terminated_length": 158.0, + "epoch": 0.8903302893232056, + "grad_norm": 1.9522093534469604, + "kl": 6.46875, + "learning_rate": 1.3270704322144832e-07, + "loss": 0.3822, + "num_tokens": 1393086069.0, + "reward": 1.8466796875, + "reward_std": 0.49706533551216125, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.86328125, + "rewards/format_reward/std": 0.3438861668109894, + "rewards/tag_count_reward/mean": 0.9306640625, + "rewards/tag_count_reward/std": 0.19238728284835815, + "step": 2608 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1944.0, + "completions/mean_length": 824.224609375, + "completions/mean_terminated_length": 784.7479858398438, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "epoch": 0.8906716736365964, + "grad_norm": 1.0797889232635498, + "kl": 7.1015625, + "learning_rate": 1.3250668749582782e-07, + "loss": 0.4339, + "num_tokens": 1393581656.0, + "reward": 1.9013671875, + "reward_std": 0.5164161324501038, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.8828125, + "rewards/format_reward/std": 0.32195815443992615, + "rewards/tag_count_reward/mean": 0.9384765625, + "rewards/tag_count_reward/std": 0.18339595198631287, + "step": 2609 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2015.0, + "completions/mean_length": 807.873046875, + "completions/mean_terminated_length": 783.1693725585938, + "completions/min_length": 141.0, + "completions/min_terminated_length": 141.0, + "epoch": 0.8910130579499872, + "grad_norm": 1.6566232442855835, + "kl": 5.03125, + "learning_rate": 1.3230692432560403e-07, + "loss": 0.3093, + "num_tokens": 1394069015.0, + "reward": 1.88427734375, + "reward_std": 0.5347901582717896, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.94091796875, + "rewards/tag_count_reward/std": 0.16640710830688477, + "step": 2610 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 825.880859375, + "completions/mean_terminated_length": 806.482177734375, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "epoch": 0.891354442263378, + "grad_norm": 1.545709490776062, + "kl": 5.796875, + "learning_rate": 1.321077539943039e-07, + "loss": 0.3447, + "num_tokens": 1394574554.0, + "reward": 1.861328125, + "reward_std": 0.5165987014770508, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.93359375, + "rewards/tag_count_reward/std": 0.18567688763141632, + "step": 2611 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1865.0, + "completions/mean_length": 823.744140625, + "completions/mean_terminated_length": 786.7947387695312, + "completions/min_length": 109.0, + "completions/min_terminated_length": 109.0, + "epoch": 0.8916958265767688, + "grad_norm": 0.9597628712654114, + "kl": 6.40625, + "learning_rate": 1.319091767846136e-07, + "loss": 0.3888, + "num_tokens": 1395068855.0, + "reward": 1.8359375, + "reward_std": 0.5429961681365967, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.923828125, + "rewards/tag_count_reward/std": 0.1992122381925583, + "step": 2612 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2022.0, + "completions/mean_length": 832.119140625, + "completions/mean_terminated_length": 785.2596435546875, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "epoch": 0.8920372108901596, + "grad_norm": 2.148956060409546, + "kl": 8.0625, + "learning_rate": 1.3171119297837686e-07, + "loss": 0.4453, + "num_tokens": 1395578452.0, + "reward": 1.7353515625, + "reward_std": 0.5739935040473938, + "rewards/accuracy_reward/mean": 0.025390625, + "rewards/accuracy_reward/std": 0.15746226906776428, + "rewards/format_reward/mean": 0.802734375, + "rewards/format_reward/std": 0.3983237147331238, + "rewards/tag_count_reward/mean": 0.9072265625, + "rewards/tag_count_reward/std": 0.21258479356765747, + "step": 2613 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 846.14453125, + "completions/mean_terminated_length": 804.8687133789062, + "completions/min_length": 122.0, + "completions/min_terminated_length": 122.0, + "epoch": 0.8923785952035505, + "grad_norm": 1.870068907737732, + "kl": 8.015625, + "learning_rate": 1.3151380285659565e-07, + "loss": 0.4641, + "num_tokens": 1396087358.0, + "reward": 1.77880859375, + "reward_std": 0.5826443433761597, + "rewards/accuracy_reward/mean": 0.05645161122083664, + "rewards/accuracy_reward/std": 0.23102475702762604, + "rewards/format_reward/mean": 0.814453125, + "rewards/format_reward/std": 0.38912075757980347, + "rewards/tag_count_reward/mean": 0.90966796875, + "rewards/tag_count_reward/std": 0.21162240207195282, + "step": 2614 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1991.0, + "completions/mean_length": 778.541015625, + "completions/mean_terminated_length": 734.9434814453125, + "completions/min_length": 7.0, + "completions/min_terminated_length": 7.0, + "epoch": 0.8927199795169412, + "grad_norm": 1.4676388502120972, + "kl": 6.90625, + "learning_rate": 1.3131700669942907e-07, + "loss": 0.4164, + "num_tokens": 1396562211.0, + "reward": 1.8330078125, + "reward_std": 0.5347275733947754, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.8359375, + "rewards/format_reward/std": 0.37069445848464966, + "rewards/tag_count_reward/mean": 0.9169921875, + "rewards/tag_count_reward/std": 0.2055833637714386, + "step": 2615 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2001.0, + "completions/mean_length": 929.205078125, + "completions/mean_terminated_length": 859.570556640625, + "completions/min_length": 119.0, + "completions/min_terminated_length": 119.0, + "epoch": 0.893061363830332, + "grad_norm": 1.4115605354309082, + "kl": 6.88671875, + "learning_rate": 1.3112080478619333e-07, + "loss": 0.4301, + "num_tokens": 1397121068.0, + "reward": 1.77197265625, + "reward_std": 0.5459601879119873, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.81640625, + "rewards/format_reward/std": 0.3875311613082886, + "rewards/tag_count_reward/mean": 0.91064453125, + "rewards/tag_count_reward/std": 0.20380185544490814, + "step": 2616 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2030.0, + "completions/mean_length": 818.673828125, + "completions/mean_terminated_length": 768.701171875, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "epoch": 0.8934027481437228, + "grad_norm": 2.140897035598755, + "kl": 6.7109375, + "learning_rate": 1.309251973953612e-07, + "loss": 0.4308, + "num_tokens": 1397615061.0, + "reward": 1.8173828125, + "reward_std": 0.5434540510177612, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.9248046875, + "rewards/tag_count_reward/std": 0.19335831701755524, + "step": 2617 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1965.0, + "completions/mean_length": 806.73046875, + "completions/mean_terminated_length": 766.6895141601562, + "completions/min_length": 70.0, + "completions/min_terminated_length": 70.0, + "epoch": 0.8937441324571136, + "grad_norm": 1.4841294288635254, + "kl": 5.3203125, + "learning_rate": 1.3073018480456148e-07, + "loss": 0.3235, + "num_tokens": 1398107899.0, + "reward": 1.873046875, + "reward_std": 0.5673485994338989, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.849609375, + "rewards/format_reward/std": 0.35780346393585205, + "rewards/tag_count_reward/mean": 0.92578125, + "rewards/tag_count_reward/std": 0.19933690130710602, + "step": 2618 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1928.0, + "completions/mean_length": 863.703125, + "completions/mean_terminated_length": 813.0509643554688, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "epoch": 0.8940855167705044, + "grad_norm": 2.4232535362243652, + "kl": 5.84375, + "learning_rate": 1.3053576729057902e-07, + "loss": 0.3975, + "num_tokens": 1398631731.0, + "reward": 1.80712890625, + "reward_std": 0.5566418766975403, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.91845703125, + "rewards/tag_count_reward/std": 0.21288292109966278, + "step": 2619 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1956.0, + "completions/mean_length": 868.373046875, + "completions/mean_terminated_length": 825.3906860351562, + "completions/min_length": 29.0, + "completions/min_terminated_length": 29.0, + "epoch": 0.8944269010838952, + "grad_norm": 2.421449899673462, + "kl": 4.34765625, + "learning_rate": 1.3034194512935377e-07, + "loss": 0.2908, + "num_tokens": 1399159682.0, + "reward": 1.87890625, + "reward_std": 0.497196763753891, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.873046875, + "rewards/format_reward/std": 0.33324605226516724, + "rewards/tag_count_reward/mean": 0.935546875, + "rewards/tag_count_reward/std": 0.18438583612442017, + "step": 2620 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1992.0, + "completions/mean_length": 814.171875, + "completions/mean_terminated_length": 771.7980346679688, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "epoch": 0.894768285397286, + "grad_norm": 2.9884707927703857, + "kl": 5.0703125, + "learning_rate": 1.3014871859598092e-07, + "loss": 0.3511, + "num_tokens": 1399656250.0, + "reward": 1.81640625, + "reward_std": 0.5273551940917969, + "rewards/accuracy_reward/mean": 0.029296875, + "rewards/accuracy_reward/std": 0.16880230605602264, + "rewards/format_reward/mean": 0.859375, + "rewards/format_reward/std": 0.3479743003845215, + "rewards/tag_count_reward/mean": 0.927734375, + "rewards/tag_count_reward/std": 0.19821202754974365, + "step": 2621 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2015.0, + "completions/mean_length": 817.431640625, + "completions/mean_terminated_length": 780.291748046875, + "completions/min_length": 135.0, + "completions/min_terminated_length": 135.0, + "epoch": 0.8951096697106768, + "grad_norm": 3.455577850341797, + "kl": 4.4453125, + "learning_rate": 1.299560879647101e-07, + "loss": 0.3186, + "num_tokens": 1400158871.0, + "reward": 1.9052734375, + "reward_std": 0.48785555362701416, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.892578125, + "rewards/format_reward/std": 0.30995169281959534, + "rewards/tag_count_reward/mean": 0.9423828125, + "rewards/tag_count_reward/std": 0.17928588390350342, + "step": 2622 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2021.0, + "completions/mean_length": 891.88671875, + "completions/mean_terminated_length": 852.181884765625, + "completions/min_length": 123.0, + "completions/min_terminated_length": 123.0, + "epoch": 0.8954510540240675, + "grad_norm": 2.8744022846221924, + "kl": 4.96484375, + "learning_rate": 1.2976405350894536e-07, + "loss": 0.2987, + "num_tokens": 1400691645.0, + "reward": 1.83837890625, + "reward_std": 0.5029296278953552, + "rewards/accuracy_reward/mean": 0.037109375, + "rewards/accuracy_reward/std": 0.18921469151973724, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.93017578125, + "rewards/tag_count_reward/std": 0.18997004628181458, + "step": 2623 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2031.0, + "completions/mean_length": 862.896484375, + "completions/mean_terminated_length": 809.687744140625, + "completions/min_length": 75.0, + "completions/min_terminated_length": 75.0, + "epoch": 0.8957924383374584, + "grad_norm": 1.500162959098816, + "kl": 6.421875, + "learning_rate": 1.295726155012445e-07, + "loss": 0.4093, + "num_tokens": 1401207592.0, + "reward": 1.81103515625, + "reward_std": 0.5551319122314453, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.92041015625, + "rewards/tag_count_reward/std": 0.20604471862316132, + "step": 2624 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1983.0, + "completions/mean_length": 809.501953125, + "completions/mean_terminated_length": 784.8306884765625, + "completions/min_length": 130.0, + "completions/min_terminated_length": 130.0, + "epoch": 0.8961338226508492, + "grad_norm": 1.9419004917144775, + "kl": 4.03515625, + "learning_rate": 1.2938177421331875e-07, + "loss": 0.2319, + "num_tokens": 1401699017.0, + "reward": 1.904296875, + "reward_std": 0.4911288022994995, + "rewards/accuracy_reward/mean": 0.07459677755832672, + "rewards/accuracy_reward/std": 0.263004869222641, + "rewards/format_reward/mean": 0.888671875, + "rewards/format_reward/std": 0.31484565138816833, + "rewards/tag_count_reward/mean": 0.943359375, + "rewards/tag_count_reward/std": 0.17051447927951813, + "step": 2625 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1999.0, + "completions/mean_length": 895.08203125, + "completions/mean_terminated_length": 835.8973388671875, + "completions/min_length": 101.0, + "completions/min_terminated_length": 101.0, + "epoch": 0.89647520696424, + "grad_norm": 0.9074051976203918, + "kl": 7.3515625, + "learning_rate": 1.2919152991603235e-07, + "loss": 0.4617, + "num_tokens": 1402241091.0, + "reward": 1.83154296875, + "reward_std": 0.5970633625984192, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.830078125, + "rewards/format_reward/std": 0.3759314715862274, + "rewards/tag_count_reward/mean": 0.91748046875, + "rewards/tag_count_reward/std": 0.203689306974411, + "step": 2626 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 853.10546875, + "completions/mean_terminated_length": 794.3401489257812, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "epoch": 0.8968165912776308, + "grad_norm": 2.1534411907196045, + "kl": 7.0859375, + "learning_rate": 1.2900188287940223e-07, + "loss": 0.4383, + "num_tokens": 1402752745.0, + "reward": 1.85888671875, + "reward_std": 0.5010073781013489, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.92138671875, + "rewards/tag_count_reward/std": 0.19918283820152283, + "step": 2627 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1980.0, + "completions/mean_length": 884.89453125, + "completions/mean_terminated_length": 807.3541870117188, + "completions/min_length": 140.0, + "completions/min_terminated_length": 140.0, + "epoch": 0.8971579755910216, + "grad_norm": 2.211641311645508, + "kl": 8.9375, + "learning_rate": 1.2881283337259784e-07, + "loss": 0.551, + "num_tokens": 1403279379.0, + "reward": 1.81591796875, + "reward_std": 0.5615724921226501, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.90966796875, + "rewards/tag_count_reward/std": 0.2201211303472519, + "step": 2628 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1759.0, + "completions/mean_length": 865.732421875, + "completions/mean_terminated_length": 792.1473388671875, + "completions/min_length": 84.0, + "completions/min_terminated_length": 84.0, + "epoch": 0.8974993599044124, + "grad_norm": 2.0325682163238525, + "kl": 7.65625, + "learning_rate": 1.2862438166394022e-07, + "loss": 0.4941, + "num_tokens": 1403793706.0, + "reward": 1.85595703125, + "reward_std": 0.5213677883148193, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.92626953125, + "rewards/tag_count_reward/std": 0.19921162724494934, + "step": 2629 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 832.08984375, + "completions/mean_terminated_length": 780.0855712890625, + "completions/min_length": 168.0, + "completions/min_terminated_length": 168.0, + "epoch": 0.8978407442178032, + "grad_norm": 1.362596869468689, + "kl": 7.0, + "learning_rate": 1.284365280209022e-07, + "loss": 0.451, + "num_tokens": 1404303496.0, + "reward": 1.85205078125, + "reward_std": 0.530191957950592, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.857421875, + "rewards/format_reward/std": 0.3499840497970581, + "rewards/tag_count_reward/mean": 0.92431640625, + "rewards/tag_count_reward/std": 0.20153403282165527, + "step": 2630 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2005.0, + "completions/mean_length": 888.017578125, + "completions/mean_terminated_length": 830.96923828125, + "completions/min_length": 154.0, + "completions/min_terminated_length": 154.0, + "epoch": 0.898182128531194, + "grad_norm": 1.9809017181396484, + "kl": 8.359375, + "learning_rate": 1.2824927271010777e-07, + "loss": 0.5198, + "num_tokens": 1404833553.0, + "reward": 1.8310546875, + "reward_std": 0.5532511472702026, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.9208984375, + "rewards/tag_count_reward/std": 0.19929614663124084, + "step": 2631 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1741.0, + "completions/mean_length": 855.234375, + "completions/mean_terminated_length": 814.270751953125, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.8985235128445848, + "grad_norm": 2.1450774669647217, + "kl": 4.640625, + "learning_rate": 1.2806261599733127e-07, + "loss": 0.2875, + "num_tokens": 1405350697.0, + "reward": 1.876953125, + "reward_std": 0.49398133158683777, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.8828125, + "rewards/format_reward/std": 0.32195815443992615, + "rewards/tag_count_reward/mean": 0.943359375, + "rewards/tag_count_reward/std": 0.17051447927951813, + "step": 2632 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1926.0, + "completions/mean_length": 832.533203125, + "completions/mean_terminated_length": 780.5479125976562, + "completions/min_length": 22.0, + "completions/min_terminated_length": 22.0, + "epoch": 0.8988648971579756, + "grad_norm": 1.1332107782363892, + "kl": 6.3203125, + "learning_rate": 1.278765581474981e-07, + "loss": 0.3981, + "num_tokens": 1405855274.0, + "reward": 1.81982421875, + "reward_std": 0.5369333028793335, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.841796875, + "rewards/format_reward/std": 0.36528825759887695, + "rewards/tag_count_reward/mean": 0.91943359375, + "rewards/tag_count_reward/std": 0.20085012912750244, + "step": 2633 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 847.32421875, + "completions/mean_terminated_length": 788.2745361328125, + "completions/min_length": 118.0, + "completions/min_terminated_length": 118.0, + "epoch": 0.8992062814713664, + "grad_norm": 1.9365277290344238, + "kl": 7.2734375, + "learning_rate": 1.276910994246831e-07, + "loss": 0.4488, + "num_tokens": 1406364304.0, + "reward": 1.8359375, + "reward_std": 0.5888235569000244, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.9140625, + "rewards/tag_count_reward/std": 0.2085207849740982, + "step": 2634 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1982.0, + "completions/mean_length": 823.33984375, + "completions/mean_terminated_length": 760.4722900390625, + "completions/min_length": 122.0, + "completions/min_terminated_length": 122.0, + "epoch": 0.8995476657847572, + "grad_norm": 1.1484029293060303, + "kl": 8.203125, + "learning_rate": 1.275062400921112e-07, + "loss": 0.5561, + "num_tokens": 1406859966.0, + "reward": 1.83642578125, + "reward_std": 0.5626037120819092, + "rewards/accuracy_reward/mean": 0.087890625, + "rewards/accuracy_reward/std": 0.2834126651287079, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.91064453125, + "rewards/tag_count_reward/std": 0.2166028767824173, + "step": 2635 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 911.73828125, + "completions/mean_terminated_length": 853.4086303710938, + "completions/min_length": 128.0, + "completions/min_terminated_length": 128.0, + "epoch": 0.899889050098148, + "grad_norm": 1.732434868812561, + "kl": 8.2890625, + "learning_rate": 1.273219804121562e-07, + "loss": 0.5103, + "num_tokens": 1407409512.0, + "reward": 1.74755859375, + "reward_std": 0.6244097948074341, + "rewards/accuracy_reward/mean": 0.052419353276491165, + "rewards/accuracy_reward/std": 0.22309619188308716, + "rewards/format_reward/mean": 0.802734375, + "rewards/format_reward/std": 0.3983237147331238, + "rewards/tag_count_reward/mean": 0.89404296875, + "rewards/tag_count_reward/std": 0.23642945289611816, + "step": 2636 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2021.0, + "completions/mean_length": 801.1015625, + "completions/mean_terminated_length": 760.8790283203125, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "epoch": 0.9002304344115388, + "grad_norm": 1.1615513563156128, + "kl": 6.4296875, + "learning_rate": 1.2713832064634125e-07, + "loss": 0.4347, + "num_tokens": 1407901244.0, + "reward": 1.85888671875, + "reward_std": 0.5226007103919983, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.92333984375, + "rewards/tag_count_reward/std": 0.19561529159545898, + "step": 2637 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.056640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1806.0, + "completions/mean_length": 892.8203125, + "completions/mean_terminated_length": 823.4617309570312, + "completions/min_length": 163.0, + "completions/min_terminated_length": 163.0, + "epoch": 0.9005718187249296, + "grad_norm": 2.583146810531616, + "kl": 8.6328125, + "learning_rate": 1.2695526105533768e-07, + "loss": 0.5558, + "num_tokens": 1408437136.0, + "reward": 1.748046875, + "reward_std": 0.6300433874130249, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.80859375, + "rewards/format_reward/std": 0.3937928080558777, + "rewards/tag_count_reward/mean": 0.892578125, + "rewards/tag_count_reward/std": 0.230254665017128, + "step": 2638 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1980.0, + "completions/mean_length": 857.107421875, + "completions/mean_terminated_length": 793.3970947265625, + "completions/min_length": 3.0, + "completions/min_terminated_length": 3.0, + "epoch": 0.9009132030383203, + "grad_norm": 2.558863878250122, + "kl": 7.4765625, + "learning_rate": 1.26772801898965e-07, + "loss": 0.4564, + "num_tokens": 1408957511.0, + "reward": 1.79052734375, + "reward_std": 0.602996826171875, + "rewards/accuracy_reward/mean": 0.08669354766607285, + "rewards/accuracy_reward/std": 0.281669557094574, + "rewards/format_reward/mean": 0.80859375, + "rewards/format_reward/std": 0.3937928080558777, + "rewards/tag_count_reward/mean": 0.89794921875, + "rewards/tag_count_reward/std": 0.2249389886856079, + "step": 2639 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2004.0, + "completions/mean_length": 822.44921875, + "completions/mean_terminated_length": 775.217041015625, + "completions/min_length": 119.0, + "completions/min_terminated_length": 119.0, + "epoch": 0.9012545873517112, + "grad_norm": 2.6348209381103516, + "kl": 5.765625, + "learning_rate": 1.2659094343619087e-07, + "loss": 0.3555, + "num_tokens": 1409460893.0, + "reward": 1.85498046875, + "reward_std": 0.5454111695289612, + "rewards/accuracy_reward/mean": 0.07056451588869095, + "rewards/accuracy_reward/std": 0.25635457038879395, + "rewards/format_reward/mean": 0.859375, + "rewards/format_reward/std": 0.3479743003845215, + "rewards/tag_count_reward/mean": 0.92724609375, + "rewards/tag_count_reward/std": 0.1875656098127365, + "step": 2640 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 764.1015625, + "completions/mean_terminated_length": 743.7222900390625, + "completions/min_length": 110.0, + "completions/min_terminated_length": 110.0, + "epoch": 0.901595971665102, + "grad_norm": 2.027493953704834, + "kl": 4.8671875, + "learning_rate": 1.2640968592512978e-07, + "loss": 0.2933, + "num_tokens": 1409927345.0, + "reward": 1.89453125, + "reward_std": 0.48001429438591003, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.876953125, + "rewards/format_reward/std": 0.32881227135658264, + "rewards/tag_count_reward/mean": 0.9375, + "rewards/tag_count_reward/std": 0.17556172609329224, + "step": 2641 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2012.0, + "completions/mean_length": 898.37109375, + "completions/mean_terminated_length": 841.8319091796875, + "completions/min_length": 125.0, + "completions/min_terminated_length": 125.0, + "epoch": 0.9019373559784928, + "grad_norm": 2.3642499446868896, + "kl": 5.40625, + "learning_rate": 1.2622902962304394e-07, + "loss": 0.3667, + "num_tokens": 1410466095.0, + "reward": 1.8291015625, + "reward_std": 0.5632820725440979, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.849609375, + "rewards/format_reward/std": 0.35780346393585205, + "rewards/tag_count_reward/mean": 0.9208984375, + "rewards/tag_count_reward/std": 0.20294499397277832, + "step": 2642 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2009.0, + "completions/mean_length": 772.806640625, + "completions/mean_terminated_length": 742.2020263671875, + "completions/min_length": 107.0, + "completions/min_terminated_length": 107.0, + "epoch": 0.9022787402918836, + "grad_norm": 2.121507406234741, + "kl": 3.2578125, + "learning_rate": 1.2604897478634172e-07, + "loss": 0.2356, + "num_tokens": 1410945996.0, + "reward": 1.92431640625, + "reward_std": 0.4335697591304779, + "rewards/accuracy_reward/mean": 0.07459677755832672, + "rewards/accuracy_reward/std": 0.263004869222641, + "rewards/format_reward/mean": 0.904296875, + "rewards/format_reward/std": 0.2944713830947876, + "rewards/tag_count_reward/mean": 0.94775390625, + "rewards/tag_count_reward/std": 0.16722622513771057, + "step": 2643 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2010.0, + "completions/mean_length": 883.7265625, + "completions/mean_terminated_length": 826.4671630859375, + "completions/min_length": 107.0, + "completions/min_terminated_length": 107.0, + "epoch": 0.9026201246052744, + "grad_norm": 2.303095817565918, + "kl": 5.3984375, + "learning_rate": 1.2586952167057805e-07, + "loss": 0.3467, + "num_tokens": 1411479440.0, + "reward": 1.833984375, + "reward_std": 0.5213505029678345, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.857421875, + "rewards/format_reward/std": 0.3499840497970581, + "rewards/tag_count_reward/mean": 0.931640625, + "rewards/tag_count_reward/std": 0.19337067008018494, + "step": 2644 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2014.0, + "completions/mean_length": 750.623046875, + "completions/mean_terminated_length": 700.6226806640625, + "completions/min_length": 76.0, + "completions/min_terminated_length": 76.0, + "epoch": 0.9029615089186652, + "grad_norm": 4.877878665924072, + "kl": 5.08984375, + "learning_rate": 1.256906705304539e-07, + "loss": 0.3845, + "num_tokens": 1411937887.0, + "reward": 1.89990234375, + "reward_std": 0.4928579032421112, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.87890625, + "rewards/format_reward/std": 0.3265552520751953, + "rewards/tag_count_reward/mean": 0.93701171875, + "rewards/tag_count_reward/std": 0.17986257374286652, + "step": 2645 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2031.0, + "completions/mean_length": 807.642578125, + "completions/mean_terminated_length": 767.6310424804688, + "completions/min_length": 113.0, + "completions/min_terminated_length": 113.0, + "epoch": 0.903302893232056, + "grad_norm": 2.0590627193450928, + "kl": 4.33984375, + "learning_rate": 1.2551242161981563e-07, + "loss": 0.2852, + "num_tokens": 1412434776.0, + "reward": 1.8896484375, + "reward_std": 0.4454382061958313, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.892578125, + "rewards/format_reward/std": 0.30995169281959534, + "rewards/tag_count_reward/mean": 0.9423828125, + "rewards/tag_count_reward/std": 0.17514485120773315, + "step": 2646 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2016.0, + "completions/mean_length": 806.576171875, + "completions/mean_terminated_length": 766.5302124023438, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "epoch": 0.9036442775454467, + "grad_norm": 3.0107901096343994, + "kl": 5.796875, + "learning_rate": 1.253347751916551e-07, + "loss": 0.407, + "num_tokens": 1412924255.0, + "reward": 1.87255859375, + "reward_std": 0.5324903130531311, + "rewards/accuracy_reward/mean": 0.07056451588869095, + "rewards/accuracy_reward/std": 0.25635457038879395, + "rewards/format_reward/mean": 0.876953125, + "rewards/format_reward/std": 0.32881227135658264, + "rewards/tag_count_reward/mean": 0.92724609375, + "rewards/tag_count_reward/std": 0.19710436463356018, + "step": 2647 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2004.0, + "completions/mean_length": 796.70703125, + "completions/mean_terminated_length": 764.1082153320312, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "epoch": 0.9039856618588376, + "grad_norm": 1.3113056421279907, + "kl": 4.96875, + "learning_rate": 1.2515773149810875e-07, + "loss": 0.3184, + "num_tokens": 1413414345.0, + "reward": 1.91650390625, + "reward_std": 0.3993160128593445, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.904296875, + "rewards/format_reward/std": 0.2944713830947876, + "rewards/tag_count_reward/mean": 0.95751953125, + "rewards/tag_count_reward/std": 0.15172556042671204, + "step": 2648 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1917.0, + "completions/mean_length": 811.69140625, + "completions/mean_terminated_length": 774.3782348632812, + "completions/min_length": 52.0, + "completions/min_terminated_length": 52.0, + "epoch": 0.9043270461722284, + "grad_norm": 1.8289512395858765, + "kl": 5.078125, + "learning_rate": 1.24981290790458e-07, + "loss": 0.3301, + "num_tokens": 1413912075.0, + "reward": 1.88134765625, + "reward_std": 0.45731788873672485, + "rewards/accuracy_reward/mean": 0.03515625, + "rewards/accuracy_reward/std": 0.1843547374010086, + "rewards/format_reward/mean": 0.89453125, + "rewards/format_reward/std": 0.3074568510055542, + "rewards/tag_count_reward/mean": 0.95166015625, + "rewards/tag_count_reward/std": 0.15477456152439117, + "step": 2649 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1893.0, + "completions/mean_length": 771.2734375, + "completions/mean_terminated_length": 735.3814697265625, + "completions/min_length": 124.0, + "completions/min_terminated_length": 124.0, + "epoch": 0.9046684304856192, + "grad_norm": 1.23770272731781, + "kl": 4.83984375, + "learning_rate": 1.2480545331912786e-07, + "loss": 0.3244, + "num_tokens": 1414379975.0, + "reward": 1.92578125, + "reward_std": 0.3980555534362793, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.904296875, + "rewards/format_reward/std": 0.2944713830947876, + "rewards/tag_count_reward/mean": 0.953125, + "rewards/tag_count_reward/std": 0.16102655231952667, + "step": 2650 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1918.0, + "completions/mean_length": 755.50390625, + "completions/mean_terminated_length": 708.408935546875, + "completions/min_length": 122.0, + "completions/min_terminated_length": 122.0, + "epoch": 0.90500981479901, + "grad_norm": 1.767143964767456, + "kl": 6.984375, + "learning_rate": 1.246302193336876e-07, + "loss": 0.4542, + "num_tokens": 1414838185.0, + "reward": 1.9462890625, + "reward_std": 0.48878249526023865, + "rewards/accuracy_reward/mean": 0.111328125, + "rewards/accuracy_reward/std": 0.31484565138816833, + "rewards/format_reward/mean": 0.892578125, + "rewards/format_reward/std": 0.30995169281959534, + "rewards/tag_count_reward/mean": 0.9423828125, + "rewards/tag_count_reward/std": 0.17653599381446838, + "step": 2651 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1946.0, + "completions/mean_length": 779.169921875, + "completions/mean_terminated_length": 727.5914306640625, + "completions/min_length": 99.0, + "completions/min_terminated_length": 99.0, + "epoch": 0.9053511991124008, + "grad_norm": 2.6416122913360596, + "kl": 7.8125, + "learning_rate": 1.2445558908284983e-07, + "loss": 0.454, + "num_tokens": 1415314688.0, + "reward": 1.86669921875, + "reward_std": 0.5072627663612366, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.873046875, + "rewards/format_reward/std": 0.33324605226516724, + "rewards/tag_count_reward/mean": 0.93310546875, + "rewards/tag_count_reward/std": 0.1922997236251831, + "step": 2652 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1875.0, + "completions/mean_length": 799.181640625, + "completions/mean_terminated_length": 753.6781616210938, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "epoch": 0.9056925834257916, + "grad_norm": 3.158607006072998, + "kl": 8.171875, + "learning_rate": 1.2428156281447017e-07, + "loss": 0.4813, + "num_tokens": 1415800573.0, + "reward": 1.8623046875, + "reward_std": 0.4951796531677246, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.880859375, + "rewards/format_reward/std": 0.32427072525024414, + "rewards/tag_count_reward/mean": 0.9365234375, + "rewards/tag_count_reward/std": 0.18930304050445557, + "step": 2653 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 766.255859375, + "completions/mean_terminated_length": 730.2228393554688, + "completions/min_length": 163.0, + "completions/min_terminated_length": 163.0, + "epoch": 0.9060339677391824, + "grad_norm": 2.059149980545044, + "kl": 7.5859375, + "learning_rate": 1.2410814077554717e-07, + "loss": 0.4717, + "num_tokens": 1416260048.0, + "reward": 1.9111328125, + "reward_std": 0.5058963894844055, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.888671875, + "rewards/format_reward/std": 0.31484565138816833, + "rewards/tag_count_reward/mean": 0.9404296875, + "rewards/tag_count_reward/std": 0.17932851612567902, + "step": 2654 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.056640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 851.5234375, + "completions/mean_terminated_length": 779.685302734375, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "epoch": 0.9063753520525731, + "grad_norm": 4.332850933074951, + "kl": 9.296875, + "learning_rate": 1.239353232122216e-07, + "loss": 0.5657, + "num_tokens": 1416776604.0, + "reward": 1.81494140625, + "reward_std": 0.5418930053710938, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.849609375, + "rewards/format_reward/std": 0.35780346393585205, + "rewards/tag_count_reward/mean": 0.91455078125, + "rewards/tag_count_reward/std": 0.21018162369728088, + "step": 2655 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1871.0, + "completions/mean_length": 754.818359375, + "completions/mean_terminated_length": 718.4638061523438, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "epoch": 0.906716736365964, + "grad_norm": 1.4998120069503784, + "kl": 5.84765625, + "learning_rate": 1.2376311036977652e-07, + "loss": 0.3438, + "num_tokens": 1417230927.0, + "reward": 1.97900390625, + "reward_std": 0.462978720664978, + "rewards/accuracy_reward/mean": 0.119140625, + "rewards/accuracy_reward/std": 0.32427072525024414, + "rewards/format_reward/mean": 0.908203125, + "rewards/format_reward/std": 0.289021372795105, + "rewards/tag_count_reward/mean": 0.95166015625, + "rewards/tag_count_reward/std": 0.15944552421569824, + "step": 2656 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1995.0, + "completions/mean_length": 797.23828125, + "completions/mean_terminated_length": 756.89111328125, + "completions/min_length": 107.0, + "completions/min_terminated_length": 107.0, + "epoch": 0.9070581206793548, + "grad_norm": 1.8390028476715088, + "kl": 7.2734375, + "learning_rate": 1.2359150249263649e-07, + "loss": 0.4555, + "num_tokens": 1417718345.0, + "reward": 1.8720703125, + "reward_std": 0.4648139178752899, + "rewards/accuracy_reward/mean": 0.0463709682226181, + "rewards/accuracy_reward/std": 0.21049949526786804, + "rewards/format_reward/mean": 0.88671875, + "rewards/format_reward/std": 0.3172462284564972, + "rewards/tag_count_reward/mean": 0.9404296875, + "rewards/tag_count_reward/std": 0.18068745732307434, + "step": 2657 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1830.0, + "completions/mean_length": 828.07421875, + "completions/mean_terminated_length": 775.898193359375, + "completions/min_length": 149.0, + "completions/min_terminated_length": 149.0, + "epoch": 0.9073995049927456, + "grad_norm": 0.9786667227745056, + "kl": 6.890625, + "learning_rate": 1.2342049982436734e-07, + "loss": 0.4399, + "num_tokens": 1418218799.0, + "reward": 1.9091796875, + "reward_std": 0.5116897821426392, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.87890625, + "rewards/format_reward/std": 0.3265552520751953, + "rewards/tag_count_reward/mean": 0.9365234375, + "rewards/tag_count_reward/std": 0.18272781372070312, + "step": 2658 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 805.41796875, + "completions/mean_terminated_length": 780.6653442382812, + "completions/min_length": 18.0, + "completions/min_terminated_length": 18.0, + "epoch": 0.9077408893061364, + "grad_norm": 1.0915343761444092, + "kl": 4.51953125, + "learning_rate": 1.2325010260767639e-07, + "loss": 0.268, + "num_tokens": 1418707653.0, + "reward": 1.94580078125, + "reward_std": 0.44406041502952576, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.90625, + "rewards/format_reward/std": 0.29176566004753113, + "rewards/tag_count_reward/mean": 0.95751953125, + "rewards/tag_count_reward/std": 0.1476399451494217, + "step": 2659 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 699.26953125, + "completions/mean_terminated_length": 669.6566772460938, + "completions/min_length": 148.0, + "completions/min_terminated_length": 148.0, + "epoch": 0.9080822736195272, + "grad_norm": 1.5793782472610474, + "kl": 5.578125, + "learning_rate": 1.2308031108441105e-07, + "loss": 0.3933, + "num_tokens": 1419137279.0, + "reward": 1.92626953125, + "reward_std": 0.4270179867744446, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.9140625, + "rewards/format_reward/std": 0.28054583072662354, + "rewards/tag_count_reward/mean": 0.95556640625, + "rewards/tag_count_reward/std": 0.15750399231910706, + "step": 2660 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 735.955078125, + "completions/mean_terminated_length": 701.7735595703125, + "completions/min_length": 117.0, + "completions/min_terminated_length": 117.0, + "epoch": 0.908423657932918, + "grad_norm": 1.0845015048980713, + "kl": 5.390625, + "learning_rate": 1.2291112549555952e-07, + "loss": 0.3535, + "num_tokens": 1419586680.0, + "reward": 1.875, + "reward_std": 0.44786369800567627, + "rewards/accuracy_reward/mean": 0.029296875, + "rewards/accuracy_reward/std": 0.16880230605602264, + "rewards/format_reward/mean": 0.900390625, + "rewards/format_reward/std": 0.29977133870124817, + "rewards/tag_count_reward/mean": 0.9453125, + "rewards/tag_count_reward/std": 0.1808803677558899, + "step": 2661 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1903.0, + "completions/mean_length": 776.365234375, + "completions/mean_terminated_length": 727.3569946289062, + "completions/min_length": 21.0, + "completions/min_terminated_length": 21.0, + "epoch": 0.9087650422463088, + "grad_norm": 1.8664003610610962, + "kl": 6.6484375, + "learning_rate": 1.2274254608124973e-07, + "loss": 0.4734, + "num_tokens": 1420062931.0, + "reward": 1.93017578125, + "reward_std": 0.509323239326477, + "rewards/accuracy_reward/mean": 0.1015625, + "rewards/accuracy_reward/std": 0.30236753821372986, + "rewards/format_reward/mean": 0.890625, + "rewards/format_reward/std": 0.31241437792778015, + "rewards/tag_count_reward/mean": 0.93798828125, + "rewards/tag_count_reward/std": 0.18289703130722046, + "step": 2662 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1828.0, + "completions/mean_length": 786.041015625, + "completions/mean_terminated_length": 742.7010498046875, + "completions/min_length": 70.0, + "completions/min_terminated_length": 70.0, + "epoch": 0.9091064265596995, + "grad_norm": 2.83722186088562, + "kl": 6.0234375, + "learning_rate": 1.2257457308074925e-07, + "loss": 0.4302, + "num_tokens": 1420544600.0, + "reward": 1.890625, + "reward_std": 0.44958245754241943, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.90234375, + "rewards/format_reward/std": 0.29713961482048035, + "rewards/tag_count_reward/mean": 0.947265625, + "rewards/tag_count_reward/std": 0.16743822395801544, + "step": 2663 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1844.0, + "completions/mean_length": 687.99609375, + "completions/mean_terminated_length": 652.5651245117188, + "completions/min_length": 62.0, + "completions/min_terminated_length": 62.0, + "epoch": 0.9094478108730903, + "grad_norm": 1.1657016277313232, + "kl": 6.20703125, + "learning_rate": 1.2240720673246515e-07, + "loss": 0.4059, + "num_tokens": 1420968918.0, + "reward": 1.947265625, + "reward_std": 0.5019833445549011, + "rewards/accuracy_reward/mean": 0.109375, + "rewards/accuracy_reward/std": 0.31241437792778015, + "rewards/format_reward/mean": 0.89453125, + "rewards/format_reward/std": 0.3074568510055542, + "rewards/tag_count_reward/mean": 0.943359375, + "rewards/tag_count_reward/std": 0.17615941166877747, + "step": 2664 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1950.0, + "completions/mean_length": 758.087890625, + "completions/mean_terminated_length": 724.4829711914062, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "epoch": 0.9097891951864812, + "grad_norm": 0.8518578410148621, + "kl": 7.3046875, + "learning_rate": 1.2224044727394326e-07, + "loss": 0.4586, + "num_tokens": 1421440899.0, + "reward": 1.833984375, + "reward_std": 0.4974585175514221, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.93359375, + "rewards/tag_count_reward/std": 0.18033012747764587, + "step": 2665 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 810.29296875, + "completions/mean_terminated_length": 762.59228515625, + "completions/min_length": 138.0, + "completions/min_terminated_length": 138.0, + "epoch": 0.910130579499872, + "grad_norm": 1.1885873079299927, + "kl": 7.0, + "learning_rate": 1.2207429494186826e-07, + "loss": 0.4449, + "num_tokens": 1421930281.0, + "reward": 1.880859375, + "reward_std": 0.45438534021377563, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.890625, + "rewards/format_reward/std": 0.31241437792778015, + "rewards/tag_count_reward/mean": 0.9453125, + "rewards/tag_count_reward/std": 0.16899466514587402, + "step": 2666 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 769.58984375, + "completions/mean_terminated_length": 720.3204956054688, + "completions/min_length": 63.0, + "completions/min_terminated_length": 63.0, + "epoch": 0.9104719638132628, + "grad_norm": 2.0983266830444336, + "kl": 9.25, + "learning_rate": 1.219087499720628e-07, + "loss": 0.5734, + "num_tokens": 1422396791.0, + "reward": 1.86279296875, + "reward_std": 0.5810860395431519, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.91943359375, + "rewards/tag_count_reward/std": 0.20802931487560272, + "step": 2667 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1937.0, + "completions/mean_length": 781.3515625, + "completions/mean_terminated_length": 737.8505249023438, + "completions/min_length": 147.0, + "completions/min_terminated_length": 147.0, + "epoch": 0.9108133481266536, + "grad_norm": 2.4403631687164307, + "kl": 7.25, + "learning_rate": 1.2174381259948785e-07, + "loss": 0.5055, + "num_tokens": 1422879531.0, + "reward": 1.85498046875, + "reward_std": 0.49316757917404175, + "rewards/accuracy_reward/mean": 0.038306452333927155, + "rewards/accuracy_reward/std": 0.19212883710861206, + "rewards/format_reward/mean": 0.8828125, + "rewards/format_reward/std": 0.32195815443992615, + "rewards/tag_count_reward/mean": 0.93505859375, + "rewards/tag_count_reward/std": 0.19169752299785614, + "step": 2668 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1948.0, + "completions/mean_length": 795.333984375, + "completions/mean_terminated_length": 747.0567626953125, + "completions/min_length": 129.0, + "completions/min_terminated_length": 129.0, + "epoch": 0.9111547324400444, + "grad_norm": 3.1153383255004883, + "kl": 7.96875, + "learning_rate": 1.2157948305824184e-07, + "loss": 0.5038, + "num_tokens": 1423376502.0, + "reward": 1.81298828125, + "reward_std": 0.5767254829406738, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.8359375, + "rewards/format_reward/std": 0.37069445848464966, + "rewards/tag_count_reward/mean": 0.91650390625, + "rewards/tag_count_reward/std": 0.20627647638320923, + "step": 2669 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1958.0, + "completions/mean_length": 791.0546875, + "completions/mean_terminated_length": 734.620361328125, + "completions/min_length": 172.0, + "completions/min_terminated_length": 172.0, + "epoch": 0.9114961167534352, + "grad_norm": 2.4565532207489014, + "kl": 8.328125, + "learning_rate": 1.2141576158156031e-07, + "loss": 0.5237, + "num_tokens": 1423860226.0, + "reward": 1.8203125, + "reward_std": 0.5129342675209045, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.916015625, + "rewards/tag_count_reward/std": 0.2099001258611679, + "step": 2670 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1884.0, + "completions/mean_length": 758.23046875, + "completions/mean_terminated_length": 711.23486328125, + "completions/min_length": 99.0, + "completions/min_terminated_length": 99.0, + "epoch": 0.9118375010668259, + "grad_norm": 1.8298970460891724, + "kl": 6.8984375, + "learning_rate": 1.2125264840181623e-07, + "loss": 0.4461, + "num_tokens": 1424323720.0, + "reward": 1.8623046875, + "reward_std": 0.4886665940284729, + "rewards/accuracy_reward/mean": 0.07258064299821854, + "rewards/accuracy_reward/std": 0.25970885157585144, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.9306640625, + "rewards/tag_count_reward/std": 0.1878843754529953, + "step": 2671 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 767.19140625, + "completions/mean_terminated_length": 736.4520263671875, + "completions/min_length": 63.0, + "completions/min_terminated_length": 63.0, + "epoch": 0.9121788853802167, + "grad_norm": 1.485329508781433, + "kl": 5.9375, + "learning_rate": 1.2109014375051868e-07, + "loss": 0.3911, + "num_tokens": 1424790618.0, + "reward": 1.88623046875, + "reward_std": 0.4452371597290039, + "rewards/accuracy_reward/mean": 0.04032257944345474, + "rewards/accuracy_reward/std": 0.19691328704357147, + "rewards/format_reward/mean": 0.8984375, + "rewards/format_reward/std": 0.30236753821372986, + "rewards/tag_count_reward/mean": 0.94873046875, + "rewards/tag_count_reward/std": 0.16679713129997253, + "step": 2672 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1992.0, + "completions/mean_length": 817.265625, + "completions/mean_terminated_length": 751.423828125, + "completions/min_length": 124.0, + "completions/min_terminated_length": 124.0, + "epoch": 0.9125202696936076, + "grad_norm": 0.9762124419212341, + "kl": 6.7890625, + "learning_rate": 1.2092824785831342e-07, + "loss": 0.4285, + "num_tokens": 1425283618.0, + "reward": 1.8515625, + "reward_std": 0.5252445340156555, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.923828125, + "rewards/tag_count_reward/std": 0.20585507154464722, + "step": 2673 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 704.01171875, + "completions/mean_terminated_length": 668.9979858398438, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.9128616540069984, + "grad_norm": 1.9770690202713013, + "kl": 6.04296875, + "learning_rate": 1.2076696095498203e-07, + "loss": 0.4296, + "num_tokens": 1425726760.0, + "reward": 1.8896484375, + "reward_std": 0.4971660077571869, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.875, + "rewards/format_reward/std": 0.3310423493385315, + "rewards/tag_count_reward/mean": 0.9384765625, + "rewards/tag_count_reward/std": 0.17660093307495117, + "step": 2674 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1918.0, + "completions/mean_length": 752.548828125, + "completions/mean_terminated_length": 716.1304931640625, + "completions/min_length": 72.0, + "completions/min_terminated_length": 72.0, + "epoch": 0.9132030383203892, + "grad_norm": 2.3720006942749023, + "kl": 4.87890625, + "learning_rate": 1.2060628326944175e-07, + "loss": 0.3497, + "num_tokens": 1426193921.0, + "reward": 1.96044921875, + "reward_std": 0.46277543902397156, + "rewards/accuracy_reward/mean": 0.11328125, + "rewards/accuracy_reward/std": 0.3172462284564972, + "rewards/format_reward/mean": 0.900390625, + "rewards/format_reward/std": 0.29977133870124817, + "rewards/tag_count_reward/mean": 0.94677734375, + "rewards/tag_count_reward/std": 0.17267994582653046, + "step": 2675 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2016.0, + "completions/mean_length": 831.55078125, + "completions/mean_terminated_length": 789.7737426757812, + "completions/min_length": 99.0, + "completions/min_terminated_length": 99.0, + "epoch": 0.91354442263378, + "grad_norm": 2.7585463523864746, + "kl": 6.359375, + "learning_rate": 1.204462150297452e-07, + "loss": 0.4475, + "num_tokens": 1426697723.0, + "reward": 1.84716796875, + "reward_std": 0.473530113697052, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.93310546875, + "rewards/tag_count_reward/std": 0.1851712167263031, + "step": 2676 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2006.0, + "completions/mean_length": 768.109375, + "completions/mean_terminated_length": 716.081298828125, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "epoch": 0.9138858069471708, + "grad_norm": 1.0900766849517822, + "kl": 6.47265625, + "learning_rate": 1.202867564630799e-07, + "loss": 0.4529, + "num_tokens": 1427177603.0, + "reward": 1.92919921875, + "reward_std": 0.5192950367927551, + "rewards/accuracy_reward/mean": 0.12890625, + "rewards/accuracy_reward/std": 0.33542385697364807, + "rewards/format_reward/mean": 0.873046875, + "rewards/format_reward/std": 0.33324605226516724, + "rewards/tag_count_reward/mean": 0.92724609375, + "rewards/tag_count_reward/std": 0.19834154844284058, + "step": 2677 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1911.0, + "completions/mean_length": 682.728515625, + "completions/mean_terminated_length": 649.9620361328125, + "completions/min_length": 2.0, + "completions/min_terminated_length": 2.0, + "epoch": 0.9142271912605616, + "grad_norm": 1.6666417121887207, + "kl": 6.5078125, + "learning_rate": 1.2012790779576833e-07, + "loss": 0.4309, + "num_tokens": 1427608872.0, + "reward": 1.9052734375, + "reward_std": 0.5004695653915405, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.88671875, + "rewards/format_reward/std": 0.3172462284564972, + "rewards/tag_count_reward/mean": 0.9404296875, + "rewards/tag_count_reward/std": 0.17237325012683868, + "step": 2678 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1909.0, + "completions/mean_length": 788.189453125, + "completions/mean_terminated_length": 747.5503540039062, + "completions/min_length": 140.0, + "completions/min_terminated_length": 140.0, + "epoch": 0.9145685755739523, + "grad_norm": 0.8680918216705322, + "kl": 5.93359375, + "learning_rate": 1.1996966925326677e-07, + "loss": 0.3804, + "num_tokens": 1428091657.0, + "reward": 1.92236328125, + "reward_std": 0.4979419708251953, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.8828125, + "rewards/format_reward/std": 0.32195815443992615, + "rewards/tag_count_reward/mean": 0.94189453125, + "rewards/tag_count_reward/std": 0.16821186244487762, + "step": 2679 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1841.0, + "completions/mean_length": 720.80859375, + "completions/mean_terminated_length": 686.2324829101562, + "completions/min_length": 143.0, + "completions/min_terminated_length": 143.0, + "epoch": 0.9149099598873431, + "grad_norm": 2.042712926864624, + "kl": 5.4140625, + "learning_rate": 1.1981204106016626e-07, + "loss": 0.3843, + "num_tokens": 1428542999.0, + "reward": 1.92919921875, + "reward_std": 0.47073543071746826, + "rewards/accuracy_reward/mean": 0.087890625, + "rewards/accuracy_reward/std": 0.2834126651287079, + "rewards/format_reward/mean": 0.896484375, + "rewards/format_reward/std": 0.30492907762527466, + "rewards/tag_count_reward/mean": 0.94482421875, + "rewards/tag_count_reward/std": 0.1699187308549881, + "step": 2680 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 740.578125, + "completions/mean_terminated_length": 714.5338745117188, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "epoch": 0.915251344200734, + "grad_norm": 1.3484684228897095, + "kl": 5.07421875, + "learning_rate": 1.196550234401909e-07, + "loss": 0.2929, + "num_tokens": 1429011535.0, + "reward": 1.87255859375, + "reward_std": 0.470352441072464, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.93701171875, + "rewards/tag_count_reward/std": 0.18456129729747772, + "step": 2681 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1896.0, + "completions/mean_length": 815.771484375, + "completions/mean_terminated_length": 760.4468994140625, + "completions/min_length": 123.0, + "completions/min_terminated_length": 123.0, + "epoch": 0.9155927285141248, + "grad_norm": 1.431976318359375, + "kl": 6.8828125, + "learning_rate": 1.194986166161986e-07, + "loss": 0.4446, + "num_tokens": 1429509402.0, + "reward": 1.8388671875, + "reward_std": 0.522997260093689, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.9306640625, + "rewards/tag_count_reward/std": 0.1878843754529953, + "step": 2682 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1938.0, + "completions/mean_length": 776.03515625, + "completions/mean_terminated_length": 748.1077880859375, + "completions/min_length": 115.0, + "completions/min_terminated_length": 115.0, + "epoch": 0.9159341128275156, + "grad_norm": 1.2878795862197876, + "kl": 5.16796875, + "learning_rate": 1.1934282081018023e-07, + "loss": 0.3696, + "num_tokens": 1429978284.0, + "reward": 1.974609375, + "reward_std": 0.48818308115005493, + "rewards/accuracy_reward/mean": 0.125, + "rewards/accuracy_reward/std": 0.3310423493385315, + "rewards/format_reward/mean": 0.900390625, + "rewards/format_reward/std": 0.29977133870124817, + "rewards/tag_count_reward/mean": 0.94921875, + "rewards/tag_count_reward/std": 0.16436253488063812, + "step": 2683 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1952.0, + "completions/mean_length": 734.421875, + "completions/mean_terminated_length": 700.200439453125, + "completions/min_length": 149.0, + "completions/min_terminated_length": 149.0, + "epoch": 0.9162754971409064, + "grad_norm": 1.3206956386566162, + "kl": 5.94921875, + "learning_rate": 1.1918763624325942e-07, + "loss": 0.3782, + "num_tokens": 1430428660.0, + "reward": 1.916015625, + "reward_std": 0.5098152160644531, + "rewards/accuracy_reward/mean": 0.08870967477560043, + "rewards/accuracy_reward/std": 0.284611314535141, + "rewards/format_reward/mean": 0.888671875, + "rewards/format_reward/std": 0.31484565138816833, + "rewards/tag_count_reward/mean": 0.94140625, + "rewards/tag_count_reward/std": 0.17828376591205597, + "step": 2684 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2013.0, + "completions/mean_length": 699.388671875, + "completions/mean_terminated_length": 667.0220336914062, + "completions/min_length": 74.0, + "completions/min_terminated_length": 74.0, + "epoch": 0.9166168814542972, + "grad_norm": 2.0574090480804443, + "kl": 5.3203125, + "learning_rate": 1.1903306313569242e-07, + "loss": 0.354, + "num_tokens": 1430859515.0, + "reward": 1.9482421875, + "reward_std": 0.4741098880767822, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.892578125, + "rewards/format_reward/std": 0.30995169281959534, + "rewards/tag_count_reward/mean": 0.9501953125, + "rewards/tag_count_reward/std": 0.15391167998313904, + "step": 2685 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2041.0, + "completions/mean_length": 828.232421875, + "completions/mean_terminated_length": 788.8850708007812, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "epoch": 0.916958265767688, + "grad_norm": 2.8428945541381836, + "kl": 6.96875, + "learning_rate": 1.1887910170686726e-07, + "loss": 0.4383, + "num_tokens": 1431354354.0, + "reward": 1.87060546875, + "reward_std": 0.49418410658836365, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.876953125, + "rewards/format_reward/std": 0.32881227135658264, + "rewards/tag_count_reward/mean": 0.93115234375, + "rewards/tag_count_reward/std": 0.19477328658103943, + "step": 2686 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 783.01953125, + "completions/mean_terminated_length": 757.8207397460938, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "epoch": 0.9172996500810787, + "grad_norm": 3.281104564666748, + "kl": 8.671875, + "learning_rate": 1.1872575217530422e-07, + "loss": 0.5114, + "num_tokens": 1431821788.0, + "reward": 1.85107421875, + "reward_std": 0.5400803089141846, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.830078125, + "rewards/format_reward/std": 0.3759314715862274, + "rewards/tag_count_reward/mean": 0.92333984375, + "rewards/tag_count_reward/std": 0.19182707369327545, + "step": 2687 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1983.0, + "completions/mean_length": 825.80859375, + "completions/mean_terminated_length": 770.9346313476562, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "epoch": 0.9176410343944695, + "grad_norm": 3.8501455783843994, + "kl": 9.75, + "learning_rate": 1.1857301475865477e-07, + "loss": 0.5899, + "num_tokens": 1432317610.0, + "reward": 1.7919921875, + "reward_std": 0.5573383569717407, + "rewards/accuracy_reward/mean": 0.03125, + "rewards/accuracy_reward/std": 0.17416280508041382, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.9169921875, + "rewards/tag_count_reward/std": 0.21086981892585754, + "step": 2688 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1999.0, + "completions/mean_length": 844.7890625, + "completions/mean_terminated_length": 795.8779907226562, + "completions/min_length": 120.0, + "completions/min_terminated_length": 120.0, + "epoch": 0.9179824187078603, + "grad_norm": 5.4735941886901855, + "kl": 9.4140625, + "learning_rate": 1.1842088967370173e-07, + "loss": 0.5074, + "num_tokens": 1432819678.0, + "reward": 1.7939453125, + "reward_std": 0.5715153813362122, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.818359375, + "rewards/format_reward/std": 0.38592514395713806, + "rewards/tag_count_reward/mean": 0.9130859375, + "rewards/tag_count_reward/std": 0.2081148475408554, + "step": 2689 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 790.75, + "completions/mean_terminated_length": 739.6422729492188, + "completions/min_length": 115.0, + "completions/min_terminated_length": 115.0, + "epoch": 0.9183238030212512, + "grad_norm": 3.8919737339019775, + "kl": 8.8125, + "learning_rate": 1.1826937713635902e-07, + "loss": 0.5271, + "num_tokens": 1433308558.0, + "reward": 1.8359375, + "reward_std": 0.5754671096801758, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.919921875, + "rewards/tag_count_reward/std": 0.20496191084384918, + "step": 2690 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 806.310546875, + "completions/mean_terminated_length": 771.403564453125, + "completions/min_length": 98.0, + "completions/min_terminated_length": 98.0, + "epoch": 0.918665187334642, + "grad_norm": 3.3554928302764893, + "kl": 7.59375, + "learning_rate": 1.1811847736167078e-07, + "loss": 0.4384, + "num_tokens": 1433792733.0, + "reward": 1.8203125, + "reward_std": 0.5457074642181396, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.826171875, + "rewards/format_reward/std": 0.3793322443962097, + "rewards/tag_count_reward/mean": 0.916015625, + "rewards/tag_count_reward/std": 0.20338943600654602, + "step": 2691 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1957.0, + "completions/mean_length": 798.337890625, + "completions/mean_terminated_length": 755.4202270507812, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "epoch": 0.9190065716480328, + "grad_norm": 2.3615872859954834, + "kl": 7.328125, + "learning_rate": 1.1796819056381175e-07, + "loss": 0.4222, + "num_tokens": 1434272154.0, + "reward": 1.85595703125, + "reward_std": 0.5467626452445984, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.91650390625, + "rewards/tag_count_reward/std": 0.207458958029747, + "step": 2692 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1924.0, + "completions/mean_length": 747.86328125, + "completions/mean_terminated_length": 719.3173828125, + "completions/min_length": 130.0, + "completions/min_terminated_length": 130.0, + "epoch": 0.9193479559614236, + "grad_norm": 2.554333209991455, + "kl": 7.6484375, + "learning_rate": 1.178185169560865e-07, + "loss": 0.4774, + "num_tokens": 1434736372.0, + "reward": 1.859375, + "reward_std": 0.5730640292167664, + "rewards/accuracy_reward/mean": 0.1015625, + "rewards/accuracy_reward/std": 0.30236753821372986, + "rewards/format_reward/mean": 0.833984375, + "rewards/format_reward/std": 0.3724585771560669, + "rewards/tag_count_reward/mean": 0.923828125, + "rewards/tag_count_reward/std": 0.18783605098724365, + "step": 2693 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 796.974609375, + "completions/mean_terminated_length": 735.44873046875, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "epoch": 0.9196893402748144, + "grad_norm": 1.866373896598816, + "kl": 5.65625, + "learning_rate": 1.1766945675092938e-07, + "loss": 0.3898, + "num_tokens": 1435225591.0, + "reward": 1.873046875, + "reward_std": 0.5167508721351624, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.927734375, + "rewards/tag_count_reward/std": 0.19384446740150452, + "step": 2694 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2008.0, + "completions/mean_length": 782.41015625, + "completions/mean_terminated_length": 733.6348876953125, + "completions/min_length": 8.0, + "completions/min_terminated_length": 8.0, + "epoch": 0.9200307245882051, + "grad_norm": 1.9280846118927002, + "kl": 6.5546875, + "learning_rate": 1.1752101015990404e-07, + "loss": 0.4142, + "num_tokens": 1435695225.0, + "reward": 1.8388671875, + "reward_std": 0.59491366147995, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.830078125, + "rewards/format_reward/std": 0.3759314715862274, + "rewards/tag_count_reward/mean": 0.9150390625, + "rewards/tag_count_reward/std": 0.20715762674808502, + "step": 2695 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2044.0, + "completions/mean_length": 826.38671875, + "completions/mean_terminated_length": 789.51708984375, + "completions/min_length": 153.0, + "completions/min_terminated_length": 153.0, + "epoch": 0.9203721089015959, + "grad_norm": 2.4268991947174072, + "kl": 6.58203125, + "learning_rate": 1.1737317739370323e-07, + "loss": 0.441, + "num_tokens": 1436199871.0, + "reward": 1.82275390625, + "reward_std": 0.5685132145881653, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.833984375, + "rewards/format_reward/std": 0.3724585771560669, + "rewards/tag_count_reward/mean": 0.91259765625, + "rewards/tag_count_reward/std": 0.20937539637088776, + "step": 2696 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1991.0, + "completions/mean_length": 780.99609375, + "completions/mean_terminated_length": 732.1663208007812, + "completions/min_length": 108.0, + "completions/min_terminated_length": 108.0, + "epoch": 0.9207134932149867, + "grad_norm": 2.5049359798431396, + "kl": 5.90625, + "learning_rate": 1.172259586621487e-07, + "loss": 0.4165, + "num_tokens": 1436681549.0, + "reward": 1.81884765625, + "reward_std": 0.5418437123298645, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.92041015625, + "rewards/tag_count_reward/std": 0.20545026659965515, + "step": 2697 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1910.0, + "completions/mean_length": 821.501953125, + "completions/mean_terminated_length": 784.48486328125, + "completions/min_length": 71.0, + "completions/min_terminated_length": 71.0, + "epoch": 0.9210548775283776, + "grad_norm": 1.1954436302185059, + "kl": 5.3359375, + "learning_rate": 1.170793541741903e-07, + "loss": 0.3431, + "num_tokens": 1437175518.0, + "reward": 1.86865234375, + "reward_std": 0.5039311051368713, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.86328125, + "rewards/format_reward/std": 0.3438861668109894, + "rewards/tag_count_reward/mean": 0.92919921875, + "rewards/tag_count_reward/std": 0.18960754573345184, + "step": 2698 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1988.0, + "completions/mean_length": 803.15625, + "completions/mean_terminated_length": 749.9144897460938, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "epoch": 0.9213962618417684, + "grad_norm": 1.1793044805526733, + "kl": 7.30859375, + "learning_rate": 1.169333641379065e-07, + "loss": 0.4721, + "num_tokens": 1437670638.0, + "reward": 1.7958984375, + "reward_std": 0.5920631885528564, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.8203125, + "rewards/format_reward/std": 0.38430243730545044, + "rewards/tag_count_reward/mean": 0.9111328125, + "rewards/tag_count_reward/std": 0.2043152004480362, + "step": 2699 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1977.0, + "completions/mean_length": 787.669921875, + "completions/mean_terminated_length": 739.0973510742188, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "epoch": 0.9217376461551592, + "grad_norm": 1.351790428161621, + "kl": 7.28125, + "learning_rate": 1.167879887605032e-07, + "loss": 0.4765, + "num_tokens": 1438154133.0, + "reward": 1.787109375, + "reward_std": 0.5768707990646362, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.828125, + "rewards/format_reward/std": 0.3776407241821289, + "rewards/tag_count_reward/mean": 0.908203125, + "rewards/tag_count_reward/std": 0.21699129045009613, + "step": 2700 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1887.0, + "completions/mean_length": 772.0234375, + "completions/mean_terminated_length": 728.2020263671875, + "completions/min_length": 123.0, + "completions/min_terminated_length": 123.0, + "epoch": 0.92207903046855, + "grad_norm": 1.2594962120056152, + "kl": 6.8046875, + "learning_rate": 1.1664322824831437e-07, + "loss": 0.4079, + "num_tokens": 1438622721.0, + "reward": 1.80126953125, + "reward_std": 0.5574404001235962, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.82421875, + "rewards/format_reward/std": 0.3810062110424042, + "rewards/tag_count_reward/mean": 0.91650390625, + "rewards/tag_count_reward/std": 0.19656065106391907, + "step": 2701 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1986.0, + "completions/mean_length": 830.994140625, + "completions/mean_terminated_length": 789.197998046875, + "completions/min_length": 25.0, + "completions/min_terminated_length": 25.0, + "epoch": 0.9224204147819408, + "grad_norm": 1.5090302228927612, + "kl": 5.8515625, + "learning_rate": 1.1649908280680094e-07, + "loss": 0.363, + "num_tokens": 1439121950.0, + "reward": 1.84423828125, + "reward_std": 0.5673857927322388, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.92626953125, + "rewards/tag_count_reward/std": 0.19360709190368652, + "step": 2702 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 805.22265625, + "completions/mean_terminated_length": 762.5414428710938, + "completions/min_length": 83.0, + "completions/min_terminated_length": 83.0, + "epoch": 0.9227617990953315, + "grad_norm": 0.9459572434425354, + "kl": 7.421875, + "learning_rate": 1.1635555264055105e-07, + "loss": 0.4642, + "num_tokens": 1439611888.0, + "reward": 1.8203125, + "reward_std": 0.591699481010437, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.9140625, + "rewards/tag_count_reward/std": 0.21201092004776, + "step": 2703 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 734.56640625, + "completions/mean_terminated_length": 705.728515625, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "epoch": 0.9231031834087223, + "grad_norm": 2.5126640796661377, + "kl": 5.015625, + "learning_rate": 1.1621263795327965e-07, + "loss": 0.3573, + "num_tokens": 1440070114.0, + "reward": 1.92041015625, + "reward_std": 0.4671206772327423, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.888671875, + "rewards/format_reward/std": 0.31484565138816833, + "rewards/tag_count_reward/mean": 0.94580078125, + "rewards/tag_count_reward/std": 0.16806410253047943, + "step": 2704 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 764.75390625, + "completions/mean_terminated_length": 733.9560546875, + "completions/min_length": 78.0, + "completions/min_terminated_length": 78.0, + "epoch": 0.9234445677221131, + "grad_norm": 1.2936022281646729, + "kl": 7.40625, + "learning_rate": 1.1607033894782782e-07, + "loss": 0.4485, + "num_tokens": 1440541268.0, + "reward": 1.81689453125, + "reward_std": 0.5451605319976807, + "rewards/accuracy_reward/mean": 0.0625, + "rewards/accuracy_reward/std": 0.2422981858253479, + "rewards/format_reward/mean": 0.8359375, + "rewards/format_reward/std": 0.37069445848464966, + "rewards/tag_count_reward/mean": 0.91845703125, + "rewards/tag_count_reward/std": 0.2010640949010849, + "step": 2705 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 809.0625, + "completions/mean_terminated_length": 766.51318359375, + "completions/min_length": 156.0, + "completions/min_terminated_length": 156.0, + "epoch": 0.923785952035504, + "grad_norm": 1.8083739280700684, + "kl": 7.9140625, + "learning_rate": 1.1592865582616306e-07, + "loss": 0.4782, + "num_tokens": 1441032900.0, + "reward": 1.87060546875, + "reward_std": 0.5810139775276184, + "rewards/accuracy_reward/mean": 0.1015625, + "rewards/accuracy_reward/std": 0.30236753821372986, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.92333984375, + "rewards/tag_count_reward/std": 0.19933150708675385, + "step": 2706 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1911.0, + "completions/mean_length": 704.283203125, + "completions/mean_terminated_length": 669.2765502929688, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "epoch": 0.9241273363488948, + "grad_norm": 1.5286200046539307, + "kl": 6.203125, + "learning_rate": 1.1578758878937856e-07, + "loss": 0.3854, + "num_tokens": 1441476325.0, + "reward": 1.92724609375, + "reward_std": 0.5413064956665039, + "rewards/accuracy_reward/mean": 0.1171875, + "rewards/accuracy_reward/std": 0.32195815443992615, + "rewards/format_reward/mean": 0.873046875, + "rewards/format_reward/std": 0.33324605226516724, + "rewards/tag_count_reward/mean": 0.93701171875, + "rewards/tag_count_reward/std": 0.18389739096164703, + "step": 2707 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2005.0, + "completions/mean_length": 746.544921875, + "completions/mean_terminated_length": 709.9578247070312, + "completions/min_length": 44.0, + "completions/min_terminated_length": 44.0, + "epoch": 0.9244687206622856, + "grad_norm": 2.0551059246063232, + "kl": 7.453125, + "learning_rate": 1.1564713803769327e-07, + "loss": 0.4896, + "num_tokens": 1441931740.0, + "reward": 1.861328125, + "reward_std": 0.5031090974807739, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.87890625, + "rewards/format_reward/std": 0.3265552520751953, + "rewards/tag_count_reward/mean": 0.935546875, + "rewards/tag_count_reward/std": 0.18961825966835022, + "step": 2708 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1998.0, + "completions/mean_length": 822.95703125, + "completions/mean_terminated_length": 767.955078125, + "completions/min_length": 115.0, + "completions/min_terminated_length": 115.0, + "epoch": 0.9248101049756764, + "grad_norm": 1.6736626625061035, + "kl": 7.71875, + "learning_rate": 1.1550730377045126e-07, + "loss": 0.4882, + "num_tokens": 1442435782.0, + "reward": 1.8505859375, + "reward_std": 0.530368447303772, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.9248046875, + "rewards/tag_count_reward/std": 0.1946192979812622, + "step": 2709 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1936.0, + "completions/mean_length": 776.63671875, + "completions/mean_terminated_length": 748.7225341796875, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "epoch": 0.9251514892890672, + "grad_norm": 1.986305594444275, + "kl": 5.0625, + "learning_rate": 1.1536808618612175e-07, + "loss": 0.3118, + "num_tokens": 1442914828.0, + "reward": 1.8603515625, + "reward_std": 0.45481055974960327, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.880859375, + "rewards/format_reward/std": 0.32427072525024414, + "rewards/tag_count_reward/mean": 0.9384765625, + "rewards/tag_count_reward/std": 0.17934982478618622, + "step": 2710 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1975.0, + "completions/mean_length": 768.765625, + "completions/mean_terminated_length": 732.8031616210938, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "epoch": 0.925492873602458, + "grad_norm": 1.7764705419540405, + "kl": 5.5390625, + "learning_rate": 1.1522948548229875e-07, + "loss": 0.362, + "num_tokens": 1443393476.0, + "reward": 1.88330078125, + "reward_std": 0.47507160902023315, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.8828125, + "rewards/format_reward/std": 0.32195815443992615, + "rewards/tag_count_reward/mean": 0.94189453125, + "rewards/tag_count_reward/std": 0.17322689294815063, + "step": 2711 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1953.0, + "completions/mean_length": 727.130859375, + "completions/mean_terminated_length": 689.9979858398438, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "epoch": 0.9258342579158487, + "grad_norm": 0.7059771418571472, + "kl": 5.984375, + "learning_rate": 1.150915018557004e-07, + "loss": 0.3872, + "num_tokens": 1443840919.0, + "reward": 1.93603515625, + "reward_std": 0.4882371127605438, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.8828125, + "rewards/format_reward/std": 0.32195815443992615, + "rewards/tag_count_reward/mean": 0.94775390625, + "rewards/tag_count_reward/std": 0.1620255559682846, + "step": 2712 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1949.0, + "completions/mean_length": 771.474609375, + "completions/mean_terminated_length": 746.0458374023438, + "completions/min_length": 142.0, + "completions/min_terminated_length": 142.0, + "epoch": 0.9261756422292395, + "grad_norm": 0.8078057765960693, + "kl": 4.859375, + "learning_rate": 1.1495413550216933e-07, + "loss": 0.3087, + "num_tokens": 1444311786.0, + "reward": 1.919921875, + "reward_std": 0.4650794267654419, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.892578125, + "rewards/format_reward/std": 0.30995169281959534, + "rewards/tag_count_reward/mean": 0.9453125, + "rewards/tag_count_reward/std": 0.16459491848945618, + "step": 2713 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2029.0, + "completions/mean_length": 798.470703125, + "completions/mean_terminated_length": 765.9178466796875, + "completions/min_length": 145.0, + "completions/min_terminated_length": 145.0, + "epoch": 0.9265170265426304, + "grad_norm": 0.7098759412765503, + "kl": 5.828125, + "learning_rate": 1.1481738661667192e-07, + "loss": 0.3736, + "num_tokens": 1444795531.0, + "reward": 1.85205078125, + "reward_std": 0.5454970002174377, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.93017578125, + "rewards/tag_count_reward/std": 0.19125337898731232, + "step": 2714 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2004.0, + "completions/mean_length": 755.12109375, + "completions/mean_terminated_length": 726.7345581054688, + "completions/min_length": 144.0, + "completions/min_terminated_length": 144.0, + "epoch": 0.9268584108560212, + "grad_norm": 1.1023526191711426, + "kl": 3.90625, + "learning_rate": 1.1468125539329826e-07, + "loss": 0.2647, + "num_tokens": 1445257993.0, + "reward": 1.94580078125, + "reward_std": 0.4347952604293823, + "rewards/accuracy_reward/mean": 0.087890625, + "rewards/accuracy_reward/std": 0.2834126651287079, + "rewards/format_reward/mean": 0.904296875, + "rewards/format_reward/std": 0.2944713830947876, + "rewards/tag_count_reward/mean": 0.95361328125, + "rewards/tag_count_reward/std": 0.15378980338573456, + "step": 2715 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1971.0, + "completions/mean_length": 780.587890625, + "completions/mean_terminated_length": 750.1700439453125, + "completions/min_length": 166.0, + "completions/min_terminated_length": 166.0, + "epoch": 0.927199795169412, + "grad_norm": 1.2092480659484863, + "kl": 4.3359375, + "learning_rate": 1.1454574202526165e-07, + "loss": 0.2947, + "num_tokens": 1445737494.0, + "reward": 1.94775390625, + "reward_std": 0.43313342332839966, + "rewards/accuracy_reward/mean": 0.08467742055654526, + "rewards/accuracy_reward/std": 0.278682142496109, + "rewards/format_reward/mean": 0.912109375, + "rewards/format_reward/std": 0.2834126651287079, + "rewards/tag_count_reward/mean": 0.95361328125, + "rewards/tag_count_reward/std": 0.15458305180072784, + "step": 2716 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 732.1015625, + "completions/mean_terminated_length": 711.21435546875, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "epoch": 0.9275411794828028, + "grad_norm": 1.404892921447754, + "kl": 4.66015625, + "learning_rate": 1.1441084670489857e-07, + "loss": 0.3157, + "num_tokens": 1446183898.0, + "reward": 1.95556640625, + "reward_std": 0.44245997071266174, + "rewards/accuracy_reward/mean": 0.107421875, + "rewards/accuracy_reward/std": 0.30995169281959534, + "rewards/format_reward/mean": 0.896484375, + "rewards/format_reward/std": 0.30492907762527466, + "rewards/tag_count_reward/mean": 0.95166015625, + "rewards/tag_count_reward/std": 0.15867657959461212, + "step": 2717 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2003.0, + "completions/mean_length": 748.970703125, + "completions/mean_terminated_length": 720.4490966796875, + "completions/min_length": 147.0, + "completions/min_terminated_length": 147.0, + "epoch": 0.9278825637961936, + "grad_norm": 2.1646533012390137, + "kl": 5.8828125, + "learning_rate": 1.1427656962366829e-07, + "loss": 0.4138, + "num_tokens": 1446652779.0, + "reward": 1.87548828125, + "reward_std": 0.49714913964271545, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.875, + "rewards/format_reward/std": 0.3310423493385315, + "rewards/tag_count_reward/mean": 0.94189453125, + "rewards/tag_count_reward/std": 0.17251938581466675, + "step": 2718 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1929.0, + "completions/mean_length": 790.986328125, + "completions/mean_terminated_length": 755.6485595703125, + "completions/min_length": 110.0, + "completions/min_terminated_length": 110.0, + "epoch": 0.9282239481095844, + "grad_norm": 1.5209766626358032, + "kl": 6.0390625, + "learning_rate": 1.1414291097215244e-07, + "loss": 0.38, + "num_tokens": 1447135812.0, + "reward": 1.8623046875, + "reward_std": 0.4664610028266907, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.87890625, + "rewards/format_reward/std": 0.3265552520751953, + "rewards/tag_count_reward/mean": 0.9423828125, + "rewards/tag_count_reward/std": 0.16874286532402039, + "step": 2719 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 776.294921875, + "completions/mean_terminated_length": 737.9134521484375, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "epoch": 0.9285653324229751, + "grad_norm": 1.9546750783920288, + "kl": 6.53125, + "learning_rate": 1.1400987094005518e-07, + "loss": 0.4213, + "num_tokens": 1447604603.0, + "reward": 1.89208984375, + "reward_std": 0.4767943024635315, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.88671875, + "rewards/format_reward/std": 0.3172462284564972, + "rewards/tag_count_reward/mean": 0.94091796875, + "rewards/tag_count_reward/std": 0.172895610332489, + "step": 2720 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1901.0, + "completions/mean_length": 806.2734375, + "completions/mean_terminated_length": 779.0099487304688, + "completions/min_length": 113.0, + "completions/min_terminated_length": 113.0, + "epoch": 0.9289067167363659, + "grad_norm": 1.4141117334365845, + "kl": 5.5546875, + "learning_rate": 1.1387744971620236e-07, + "loss": 0.3895, + "num_tokens": 1448095559.0, + "reward": 1.8623046875, + "reward_std": 0.48223400115966797, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.8828125, + "rewards/format_reward/std": 0.32195815443992615, + "rewards/tag_count_reward/mean": 0.9404296875, + "rewards/tag_count_reward/std": 0.17795921862125397, + "step": 2721 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1993.0, + "completions/mean_length": 711.4765625, + "completions/mean_terminated_length": 673.903564453125, + "completions/min_length": 63.0, + "completions/min_terminated_length": 63.0, + "epoch": 0.9292481010497567, + "grad_norm": 2.679053544998169, + "kl": 6.5, + "learning_rate": 1.137456474885418e-07, + "loss": 0.3861, + "num_tokens": 1448537531.0, + "reward": 1.943359375, + "reward_std": 0.5373616218566895, + "rewards/accuracy_reward/mean": 0.140625, + "rewards/accuracy_reward/std": 0.3479743003845215, + "rewards/format_reward/mean": 0.8671875, + "rewards/format_reward/std": 0.33970388770103455, + "rewards/tag_count_reward/mean": 0.935546875, + "rewards/tag_count_reward/std": 0.17900052666664124, + "step": 2722 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1951.0, + "completions/mean_length": 756.78515625, + "completions/mean_terminated_length": 712.4404296875, + "completions/min_length": 131.0, + "completions/min_terminated_length": 131.0, + "epoch": 0.9295894853631476, + "grad_norm": 6.534995079040527, + "kl": 9.046875, + "learning_rate": 1.136144644441426e-07, + "loss": 0.5285, + "num_tokens": 1449003133.0, + "reward": 1.86669921875, + "reward_std": 0.5419111847877502, + "rewards/accuracy_reward/mean": 0.10080645233392715, + "rewards/accuracy_reward/std": 0.30137622356414795, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.92138671875, + "rewards/tag_count_reward/std": 0.19733208417892456, + "step": 2723 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1907.0, + "completions/mean_length": 755.728515625, + "completions/mean_terminated_length": 719.3995971679688, + "completions/min_length": 114.0, + "completions/min_terminated_length": 114.0, + "epoch": 0.9299308696765384, + "grad_norm": 5.791825294494629, + "kl": 8.21875, + "learning_rate": 1.1348390076919519e-07, + "loss": 0.4888, + "num_tokens": 1449465458.0, + "reward": 1.828125, + "reward_std": 0.5473470687866211, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.921875, + "rewards/tag_count_reward/std": 0.19091396033763885, + "step": 2724 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1799.0, + "completions/mean_length": 826.548828125, + "completions/mean_terminated_length": 774.3075561523438, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "epoch": 0.9302722539899292, + "grad_norm": 4.375201225280762, + "kl": 8.8359375, + "learning_rate": 1.1335395664901071e-07, + "loss": 0.5388, + "num_tokens": 1449975883.0, + "reward": 1.8525390625, + "reward_std": 0.5686639547348022, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.9189453125, + "rewards/tag_count_reward/std": 0.2015654444694519, + "step": 2725 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1989.0, + "completions/mean_length": 831.712890625, + "completions/mean_terminated_length": 787.394775390625, + "completions/min_length": 98.0, + "completions/min_terminated_length": 98.0, + "epoch": 0.93061363830332, + "grad_norm": 3.420452833175659, + "kl": 7.640625, + "learning_rate": 1.1322463226802109e-07, + "loss": 0.4433, + "num_tokens": 1450487784.0, + "reward": 1.8232421875, + "reward_std": 0.5823479890823364, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.8203125, + "rewards/format_reward/std": 0.38430243730545044, + "rewards/tag_count_reward/mean": 0.9169921875, + "rewards/tag_count_reward/std": 0.19893142580986023, + "step": 2726 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1993.0, + "completions/mean_length": 820.052734375, + "completions/mean_terminated_length": 788.0621337890625, + "completions/min_length": 62.0, + "completions/min_terminated_length": 62.0, + "epoch": 0.9309550226167108, + "grad_norm": 1.4049073457717896, + "kl": 6.609375, + "learning_rate": 1.1309592780977867e-07, + "loss": 0.4338, + "num_tokens": 1450988467.0, + "reward": 1.859375, + "reward_std": 0.5390852689743042, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.92578125, + "rewards/tag_count_reward/std": 0.196245014667511, + "step": 2727 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1943.0, + "completions/mean_length": 789.669921875, + "completions/mean_terminated_length": 735.8513793945312, + "completions/min_length": 152.0, + "completions/min_terminated_length": 152.0, + "epoch": 0.9312964069301015, + "grad_norm": 1.4815195798873901, + "kl": 7.2265625, + "learning_rate": 1.1296784345695585e-07, + "loss": 0.4875, + "num_tokens": 1451470842.0, + "reward": 1.84814453125, + "reward_std": 0.5590066313743591, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.8359375, + "rewards/format_reward/std": 0.37069445848464966, + "rewards/tag_count_reward/mean": 0.92626953125, + "rewards/tag_count_reward/std": 0.18521249294281006, + "step": 2728 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2017.0, + "completions/mean_length": 850.826171875, + "completions/mean_terminated_length": 812.2076416015625, + "completions/min_length": 100.0, + "completions/min_terminated_length": 100.0, + "epoch": 0.9316377912434923, + "grad_norm": 1.1708332300186157, + "kl": 6.171875, + "learning_rate": 1.1284037939134502e-07, + "loss": 0.3786, + "num_tokens": 1451986945.0, + "reward": 1.87060546875, + "reward_std": 0.5346391797065735, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.86328125, + "rewards/format_reward/std": 0.3438861668109894, + "rewards/tag_count_reward/mean": 0.92529296875, + "rewards/tag_count_reward/std": 0.19823069870471954, + "step": 2729 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 796.201171875, + "completions/mean_terminated_length": 763.5891723632812, + "completions/min_length": 157.0, + "completions/min_terminated_length": 157.0, + "epoch": 0.9319791755568831, + "grad_norm": 0.9302889108657837, + "kl": 6.78125, + "learning_rate": 1.1271353579385804e-07, + "loss": 0.4129, + "num_tokens": 1452470968.0, + "reward": 1.8251953125, + "reward_std": 0.560287356376648, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.8359375, + "rewards/format_reward/std": 0.37069445848464966, + "rewards/tag_count_reward/mean": 0.9228515625, + "rewards/tag_count_reward/std": 0.1988353431224823, + "step": 2730 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2007.0, + "completions/mean_length": 849.7421875, + "completions/mean_terminated_length": 816.0562133789062, + "completions/min_length": 40.0, + "completions/min_terminated_length": 40.0, + "epoch": 0.932320559870274, + "grad_norm": 1.5970714092254639, + "kl": 5.6484375, + "learning_rate": 1.1258731284452616e-07, + "loss": 0.3587, + "num_tokens": 1452981492.0, + "reward": 1.85205078125, + "reward_std": 0.49832069873809814, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.880859375, + "rewards/format_reward/std": 0.32427072525024414, + "rewards/tag_count_reward/mean": 0.93017578125, + "rewards/tag_count_reward/std": 0.19379454851150513, + "step": 2731 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2043.0, + "completions/mean_length": 826.978515625, + "completions/mean_terminated_length": 779.9208984375, + "completions/min_length": 150.0, + "completions/min_terminated_length": 150.0, + "epoch": 0.9326619441836648, + "grad_norm": 2.426403760910034, + "kl": 6.5546875, + "learning_rate": 1.1246171072249991e-07, + "loss": 0.4547, + "num_tokens": 1453480857.0, + "reward": 1.90576171875, + "reward_std": 0.588792622089386, + "rewards/accuracy_reward/mean": 0.119140625, + "rewards/accuracy_reward/std": 0.32427072525024414, + "rewards/format_reward/mean": 0.859375, + "rewards/format_reward/std": 0.3479743003845215, + "rewards/tag_count_reward/mean": 0.92724609375, + "rewards/tag_count_reward/std": 0.1958593726158142, + "step": 2732 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 863.044921875, + "completions/mean_terminated_length": 822.3495483398438, + "completions/min_length": 166.0, + "completions/min_terminated_length": 166.0, + "epoch": 0.9330033284970556, + "grad_norm": 1.839331865310669, + "kl": 4.6171875, + "learning_rate": 1.1233672960604836e-07, + "loss": 0.298, + "num_tokens": 1453993808.0, + "reward": 1.9404296875, + "reward_std": 0.48437240719795227, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.896484375, + "rewards/format_reward/std": 0.30492907762527466, + "rewards/tag_count_reward/mean": 0.9482421875, + "rewards/tag_count_reward/std": 0.16480088233947754, + "step": 2733 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 796.869140625, + "completions/mean_terminated_length": 735.3380737304688, + "completions/min_length": 88.0, + "completions/min_terminated_length": 88.0, + "epoch": 0.9333447128104464, + "grad_norm": 1.7125744819641113, + "kl": 6.4375, + "learning_rate": 1.1221236967255949e-07, + "loss": 0.4165, + "num_tokens": 1454486061.0, + "reward": 1.8779296875, + "reward_std": 0.5492465496063232, + "rewards/accuracy_reward/mean": 0.109375, + "rewards/accuracy_reward/std": 0.31241437792778015, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.9169921875, + "rewards/tag_count_reward/std": 0.21659238636493683, + "step": 2734 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1883.0, + "completions/mean_length": 861.9921875, + "completions/mean_terminated_length": 798.543212890625, + "completions/min_length": 145.0, + "completions/min_terminated_length": 145.0, + "epoch": 0.9336860971238372, + "grad_norm": 1.4921162128448486, + "kl": 6.5390625, + "learning_rate": 1.1208863109853938e-07, + "loss": 0.431, + "num_tokens": 1455001737.0, + "reward": 1.87255859375, + "reward_std": 0.5436522960662842, + "rewards/accuracy_reward/mean": 0.07258064299821854, + "rewards/accuracy_reward/std": 0.25970885157585144, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.93115234375, + "rewards/tag_count_reward/std": 0.19726912677288055, + "step": 2735 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 845.359375, + "completions/mean_terminated_length": 804.0565795898438, + "completions/min_length": 134.0, + "completions/min_terminated_length": 134.0, + "epoch": 0.9340274814372279, + "grad_norm": 1.247635006904602, + "kl": 6.078125, + "learning_rate": 1.1196551405961232e-07, + "loss": 0.3962, + "num_tokens": 1455508961.0, + "reward": 1.91943359375, + "reward_std": 0.5020928978919983, + "rewards/accuracy_reward/mean": 0.099609375, + "rewards/accuracy_reward/std": 0.29977133870124817, + "rewards/format_reward/mean": 0.880859375, + "rewards/format_reward/std": 0.32427072525024414, + "rewards/tag_count_reward/mean": 0.93896484375, + "rewards/tag_count_reward/std": 0.1798572540283203, + "step": 2736 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1987.0, + "completions/mean_length": 838.4921875, + "completions/mean_terminated_length": 786.7617797851562, + "completions/min_length": 140.0, + "completions/min_terminated_length": 140.0, + "epoch": 0.9343688657506187, + "grad_norm": 1.1580220460891724, + "kl": 7.0859375, + "learning_rate": 1.1184301873052052e-07, + "loss": 0.4467, + "num_tokens": 1456012653.0, + "reward": 1.85205078125, + "reward_std": 0.5236070156097412, + "rewards/accuracy_reward/mean": 0.06653226166963577, + "rewards/accuracy_reward/std": 0.2494617998600006, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.92626953125, + "rewards/tag_count_reward/std": 0.19674043357372284, + "step": 2737 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1988.0, + "completions/mean_length": 830.982421875, + "completions/mean_terminated_length": 791.7237548828125, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "epoch": 0.9347102500640095, + "grad_norm": 1.6010982990264893, + "kl": 6.40625, + "learning_rate": 1.1172114528512358e-07, + "loss": 0.3609, + "num_tokens": 1456520548.0, + "reward": 1.85693359375, + "reward_std": 0.5425029397010803, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.92138671875, + "rewards/tag_count_reward/std": 0.2022298127412796, + "step": 2738 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1922.0, + "completions/mean_length": 824.310546875, + "completions/mean_terminated_length": 774.5670166015625, + "completions/min_length": 126.0, + "completions/min_terminated_length": 126.0, + "epoch": 0.9350516343774004, + "grad_norm": 1.4851871728897095, + "kl": 8.1484375, + "learning_rate": 1.115998938963986e-07, + "loss": 0.4943, + "num_tokens": 1457016883.0, + "reward": 1.83740234375, + "reward_std": 0.5193938612937927, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.93115234375, + "rewards/tag_count_reward/std": 0.1916077733039856, + "step": 2739 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2041.0, + "completions/mean_length": 784.787109375, + "completions/mean_terminated_length": 757.0518798828125, + "completions/min_length": 90.0, + "completions/min_terminated_length": 90.0, + "epoch": 0.9353930186907912, + "grad_norm": 1.6466114521026611, + "kl": 6.6484375, + "learning_rate": 1.1147926473643973e-07, + "loss": 0.4206, + "num_tokens": 1457500262.0, + "reward": 1.865234375, + "reward_std": 0.4733615517616272, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.927734375, + "rewards/tag_count_reward/std": 0.20127369463443756, + "step": 2740 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2006.0, + "completions/mean_length": 805.1640625, + "completions/mean_terminated_length": 772.7855834960938, + "completions/min_length": 126.0, + "completions/min_terminated_length": 126.0, + "epoch": 0.935734403004182, + "grad_norm": 1.5594877004623413, + "kl": 5.7890625, + "learning_rate": 1.1135925797645812e-07, + "loss": 0.3914, + "num_tokens": 1457985050.0, + "reward": 1.91943359375, + "reward_std": 0.5162756443023682, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.884765625, + "rewards/format_reward/std": 0.3196168541908264, + "rewards/tag_count_reward/mean": 0.94091796875, + "rewards/tag_count_reward/std": 0.1750049889087677, + "step": 2741 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1972.0, + "completions/mean_length": 797.59765625, + "completions/mean_terminated_length": 759.859130859375, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "epoch": 0.9360757873175728, + "grad_norm": 1.6687159538269043, + "kl": 5.03515625, + "learning_rate": 1.1123987378678127e-07, + "loss": 0.3125, + "num_tokens": 1458470924.0, + "reward": 1.90478515625, + "reward_std": 0.4768308401107788, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.896484375, + "rewards/format_reward/std": 0.30492907762527466, + "rewards/tag_count_reward/mean": 0.94384765625, + "rewards/tag_count_reward/std": 0.17527233064174652, + "step": 2742 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1958.0, + "completions/mean_length": 773.404296875, + "completions/mean_terminated_length": 755.7366333007812, + "completions/min_length": 89.0, + "completions/min_terminated_length": 89.0, + "epoch": 0.9364171716309636, + "grad_norm": 1.5119606256484985, + "kl": 4.30078125, + "learning_rate": 1.1112111233685323e-07, + "loss": 0.2805, + "num_tokens": 1458948491.0, + "reward": 1.953125, + "reward_std": 0.43765851855278015, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.919921875, + "rewards/format_reward/std": 0.271679550409317, + "rewards/tag_count_reward/mean": 0.958984375, + "rewards/tag_count_reward/std": 0.15253034234046936, + "step": 2743 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 845.7890625, + "completions/mean_terminated_length": 804.5010375976562, + "completions/min_length": 125.0, + "completions/min_terminated_length": 125.0, + "epoch": 0.9367585559443543, + "grad_norm": 1.2072147130966187, + "kl": 5.609375, + "learning_rate": 1.1100297379523423e-07, + "loss": 0.3519, + "num_tokens": 1459464143.0, + "reward": 1.951171875, + "reward_std": 0.4601624608039856, + "rewards/accuracy_reward/mean": 0.103515625, + "rewards/accuracy_reward/std": 0.30492907762527466, + "rewards/format_reward/mean": 0.896484375, + "rewards/format_reward/std": 0.30492907762527466, + "rewards/tag_count_reward/mean": 0.951171875, + "rewards/tag_count_reward/std": 0.15814046561717987, + "step": 2744 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1868.0, + "completions/mean_length": 787.537109375, + "completions/mean_terminated_length": 752.1023559570312, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "epoch": 0.9370999402577451, + "grad_norm": 1.7770161628723145, + "kl": 6.6796875, + "learning_rate": 1.108854583296002e-07, + "loss": 0.4191, + "num_tokens": 1459940114.0, + "reward": 1.88330078125, + "reward_std": 0.5040788650512695, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.93603515625, + "rewards/tag_count_reward/std": 0.18620555102825165, + "step": 2745 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 814.953125, + "completions/mean_terminated_length": 767.4320068359375, + "completions/min_length": 128.0, + "completions/min_terminated_length": 128.0, + "epoch": 0.9374413245711359, + "grad_norm": 1.4414271116256714, + "kl": 6.1171875, + "learning_rate": 1.1076856610674298e-07, + "loss": 0.3417, + "num_tokens": 1460450394.0, + "reward": 1.826171875, + "reward_std": 0.5320627689361572, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.931640625, + "rewards/tag_count_reward/std": 0.186283141374588, + "step": 2746 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2010.0, + "completions/mean_length": 867.416015625, + "completions/mean_terminated_length": 829.3326416015625, + "completions/min_length": 165.0, + "completions/min_terminated_length": 165.0, + "epoch": 0.9377827088845268, + "grad_norm": 1.8329652547836304, + "kl": 6.3359375, + "learning_rate": 1.106522972925696e-07, + "loss": 0.366, + "num_tokens": 1460977279.0, + "reward": 1.865234375, + "reward_std": 0.47398167848587036, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.875, + "rewards/format_reward/std": 0.3310423493385315, + "rewards/tag_count_reward/mean": 0.939453125, + "rewards/tag_count_reward/std": 0.18036192655563354, + "step": 2747 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1990.0, + "completions/mean_length": 873.8671875, + "completions/mean_terminated_length": 828.6166381835938, + "completions/min_length": 216.0, + "completions/min_terminated_length": 216.0, + "epoch": 0.9381240931979176, + "grad_norm": 1.8618098497390747, + "kl": 7.3359375, + "learning_rate": 1.1053665205210249e-07, + "loss": 0.4809, + "num_tokens": 1461494891.0, + "reward": 1.82421875, + "reward_std": 0.5058585405349731, + "rewards/accuracy_reward/mean": 0.01953125, + "rewards/accuracy_reward/std": 0.1385180652141571, + "rewards/format_reward/mean": 0.873046875, + "rewards/format_reward/std": 0.33324605226516724, + "rewards/tag_count_reward/mean": 0.931640625, + "rewards/tag_count_reward/std": 0.19210147857666016, + "step": 2748 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2022.0, + "completions/mean_length": 793.55078125, + "completions/mean_terminated_length": 768.561767578125, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "epoch": 0.9384654775113084, + "grad_norm": 1.2260710000991821, + "kl": 5.04296875, + "learning_rate": 1.1042163054947881e-07, + "loss": 0.3005, + "num_tokens": 1461975909.0, + "reward": 1.92529296875, + "reward_std": 0.45410025119781494, + "rewards/accuracy_reward/mean": 0.08541666716337204, + "rewards/accuracy_reward/std": 0.27979233860969543, + "rewards/format_reward/mean": 0.89453125, + "rewards/format_reward/std": 0.3074568510055542, + "rewards/tag_count_reward/mean": 0.95068359375, + "rewards/tag_count_reward/std": 0.15837518870830536, + "step": 2749 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1871.0, + "completions/mean_length": 875.1796875, + "completions/mean_terminated_length": 827.5040283203125, + "completions/min_length": 166.0, + "completions/min_terminated_length": 166.0, + "epoch": 0.9388068618246992, + "grad_norm": 1.4115797281265259, + "kl": 5.65625, + "learning_rate": 1.1030723294795055e-07, + "loss": 0.3572, + "num_tokens": 1462501089.0, + "reward": 1.83935546875, + "reward_std": 0.4400091767311096, + "rewards/accuracy_reward/mean": 0.02734375, + "rewards/accuracy_reward/std": 0.16324250400066376, + "rewards/format_reward/mean": 0.876953125, + "rewards/format_reward/std": 0.32881227135658264, + "rewards/tag_count_reward/mean": 0.93505859375, + "rewards/tag_count_reward/std": 0.1865234375, + "step": 2750 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2025.0, + "completions/mean_length": 836.431640625, + "completions/mean_terminated_length": 779.44580078125, + "completions/min_length": 60.0, + "completions/min_terminated_length": 60.0, + "epoch": 0.93914824613809, + "grad_norm": 2.2590596675872803, + "kl": 7.5234375, + "learning_rate": 1.1019345940988427e-07, + "loss": 0.4358, + "num_tokens": 1463009102.0, + "reward": 1.82275390625, + "reward_std": 0.4858624339103699, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.92041015625, + "rewards/tag_count_reward/std": 0.19631743431091309, + "step": 2751 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 906.3359375, + "completions/mean_terminated_length": 862.336669921875, + "completions/min_length": 138.0, + "completions/min_terminated_length": 138.0, + "epoch": 0.9394896304514807, + "grad_norm": 1.9747235774993896, + "kl": 6.8671875, + "learning_rate": 1.1008031009676061e-07, + "loss": 0.4171, + "num_tokens": 1463562954.0, + "reward": 1.8486328125, + "reward_std": 0.5552176833152771, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.9208984375, + "rewards/tag_count_reward/std": 0.19495287537574768, + "step": 2752 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 760.125, + "completions/mean_terminated_length": 734.4701538085938, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "epoch": 0.9398310147648715, + "grad_norm": 2.940276861190796, + "kl": 4.484375, + "learning_rate": 1.0996778516917438e-07, + "loss": 0.3332, + "num_tokens": 1464033370.0, + "reward": 1.95849609375, + "reward_std": 0.47670936584472656, + "rewards/accuracy_reward/mean": 0.107421875, + "rewards/accuracy_reward/std": 0.30995169281959534, + "rewards/format_reward/mean": 0.90234375, + "rewards/format_reward/std": 0.29713961482048035, + "rewards/tag_count_reward/mean": 0.94873046875, + "rewards/tag_count_reward/std": 0.16308951377868652, + "step": 2753 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2028.0, + "completions/mean_length": 796.9375, + "completions/mean_terminated_length": 764.3447265625, + "completions/min_length": 108.0, + "completions/min_terminated_length": 108.0, + "epoch": 0.9401723990782623, + "grad_norm": 1.0292353630065918, + "kl": 5.05859375, + "learning_rate": 1.0985588478683407e-07, + "loss": 0.3041, + "num_tokens": 1464517642.0, + "reward": 1.884765625, + "reward_std": 0.5098904371261597, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.935546875, + "rewards/tag_count_reward/std": 0.17693878710269928, + "step": 2754 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1859.0, + "completions/mean_length": 822.7265625, + "completions/mean_terminated_length": 770.3218383789062, + "completions/min_length": 114.0, + "completions/min_terminated_length": 114.0, + "epoch": 0.9405137833916531, + "grad_norm": 1.6634807586669922, + "kl": 5.89453125, + "learning_rate": 1.0974460910856182e-07, + "loss": 0.394, + "num_tokens": 1465017150.0, + "reward": 1.86083984375, + "reward_std": 0.5335161089897156, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.93115234375, + "rewards/tag_count_reward/std": 0.18773873150348663, + "step": 2755 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2015.0, + "completions/mean_length": 865.8359375, + "completions/mean_terminated_length": 810.2330932617188, + "completions/min_length": 88.0, + "completions/min_terminated_length": 88.0, + "epoch": 0.940855167705044, + "grad_norm": 1.9925856590270996, + "kl": 6.6953125, + "learning_rate": 1.0963395829229322e-07, + "loss": 0.3993, + "num_tokens": 1465539626.0, + "reward": 1.892578125, + "reward_std": 0.5888175964355469, + "rewards/accuracy_reward/mean": 0.1484375, + "rewards/accuracy_reward/std": 0.35588082671165466, + "rewards/format_reward/mean": 0.830078125, + "rewards/format_reward/std": 0.3759314715862274, + "rewards/tag_count_reward/mean": 0.9140625, + "rewards/tag_count_reward/std": 0.19891461730003357, + "step": 2756 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1988.0, + "completions/mean_length": 822.06640625, + "completions/mean_terminated_length": 787.6023559570312, + "completions/min_length": 146.0, + "completions/min_terminated_length": 146.0, + "epoch": 0.9411965520184348, + "grad_norm": 1.9566264152526855, + "kl": 4.609375, + "learning_rate": 1.0952393249507669e-07, + "loss": 0.2954, + "num_tokens": 1466041836.0, + "reward": 1.904296875, + "reward_std": 0.5156567096710205, + "rewards/accuracy_reward/mean": 0.1171875, + "rewards/accuracy_reward/std": 0.32195815443992615, + "rewards/format_reward/mean": 0.859375, + "rewards/format_reward/std": 0.3479743003845215, + "rewards/tag_count_reward/mean": 0.927734375, + "rewards/tag_count_reward/std": 0.1834714710712433, + "step": 2757 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1950.0, + "completions/mean_length": 872.8046875, + "completions/mean_terminated_length": 804.8181762695312, + "completions/min_length": 118.0, + "completions/min_terminated_length": 118.0, + "epoch": 0.9415379363318256, + "grad_norm": 2.669552803039551, + "kl": 6.8671875, + "learning_rate": 1.0941453187307386e-07, + "loss": 0.4291, + "num_tokens": 1466568136.0, + "reward": 1.779296875, + "reward_std": 0.5448940992355347, + "rewards/accuracy_reward/mean": 0.04032257944345474, + "rewards/accuracy_reward/std": 0.19691328704357147, + "rewards/format_reward/mean": 0.828125, + "rewards/format_reward/std": 0.3776407241821289, + "rewards/tag_count_reward/mean": 0.912109375, + "rewards/tag_count_reward/std": 0.20592933893203735, + "step": 2758 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.056640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 886.259765625, + "completions/mean_terminated_length": 816.5072631835938, + "completions/min_length": 152.0, + "completions/min_terminated_length": 152.0, + "epoch": 0.9418793206452164, + "grad_norm": 4.988946914672852, + "kl": 8.734375, + "learning_rate": 1.0930575658155882e-07, + "loss": 0.5076, + "num_tokens": 1467114845.0, + "reward": 1.67578125, + "reward_std": 0.6371972560882568, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.76171875, + "rewards/format_reward/std": 0.42644867300987244, + "rewards/tag_count_reward/mean": 0.87109375, + "rewards/tag_count_reward/std": 0.2381935715675354, + "step": 2759 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2014.0, + "completions/mean_length": 797.75390625, + "completions/mean_terminated_length": 749.5699462890625, + "completions/min_length": 9.0, + "completions/min_terminated_length": 9.0, + "epoch": 0.9422207049586071, + "grad_norm": 1.230309247970581, + "kl": 6.7265625, + "learning_rate": 1.0919760677491827e-07, + "loss": 0.4447, + "num_tokens": 1467599647.0, + "reward": 1.82666015625, + "reward_std": 0.5500995516777039, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.91259765625, + "rewards/tag_count_reward/std": 0.20284785330295563, + "step": 2760 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1950.0, + "completions/mean_length": 878.380859375, + "completions/mean_terminated_length": 810.7169189453125, + "completions/min_length": 150.0, + "completions/min_terminated_length": 150.0, + "epoch": 0.9425620892719979, + "grad_norm": 2.21763277053833, + "kl": 7.4765625, + "learning_rate": 1.0909008260665102e-07, + "loss": 0.4783, + "num_tokens": 1468138690.0, + "reward": 1.76171875, + "reward_std": 0.5877810120582581, + "rewards/accuracy_reward/mean": 0.033203125, + "rewards/accuracy_reward/std": 0.17934183776378632, + "rewards/format_reward/mean": 0.82421875, + "rewards/format_reward/std": 0.3810062110424042, + "rewards/tag_count_reward/mean": 0.904296875, + "rewards/tag_count_reward/std": 0.2236529439687729, + "step": 2761 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1988.0, + "completions/mean_length": 834.28125, + "completions/mean_terminated_length": 782.3707275390625, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.9429034735853887, + "grad_norm": 2.670936107635498, + "kl": 6.578125, + "learning_rate": 1.0898318422936796e-07, + "loss": 0.4466, + "num_tokens": 1468640258.0, + "reward": 1.84130859375, + "reward_std": 0.5695170164108276, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.91748046875, + "rewards/tag_count_reward/std": 0.2012730985879898, + "step": 2762 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 867.34765625, + "completions/mean_terminated_length": 816.8513793945312, + "completions/min_length": 122.0, + "completions/min_terminated_length": 122.0, + "epoch": 0.9432448578987795, + "grad_norm": 1.1016266345977783, + "kl": 5.6484375, + "learning_rate": 1.0887691179479182e-07, + "loss": 0.3679, + "num_tokens": 1469162836.0, + "reward": 1.85888671875, + "reward_std": 0.5383203625679016, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.859375, + "rewards/format_reward/std": 0.3479743003845215, + "rewards/tag_count_reward/mean": 0.92529296875, + "rewards/tag_count_reward/std": 0.19823069870471954, + "step": 2763 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2029.0, + "completions/mean_length": 818.064453125, + "completions/mean_terminated_length": 775.8242797851562, + "completions/min_length": 44.0, + "completions/min_terminated_length": 44.0, + "epoch": 0.9435862422121704, + "grad_norm": 4.187354564666748, + "kl": 4.8046875, + "learning_rate": 1.0877126545375688e-07, + "loss": 0.3482, + "num_tokens": 1469657781.0, + "reward": 1.91357421875, + "reward_std": 0.5429072976112366, + "rewards/accuracy_reward/mean": 0.119140625, + "rewards/accuracy_reward/std": 0.32427072525024414, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.92919921875, + "rewards/tag_count_reward/std": 0.18766237795352936, + "step": 2764 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1930.0, + "completions/mean_length": 818.171875, + "completions/mean_terminated_length": 762.955078125, + "completions/min_length": 110.0, + "completions/min_terminated_length": 110.0, + "epoch": 0.9439276265255612, + "grad_norm": 2.6798832416534424, + "kl": 5.875, + "learning_rate": 1.0866624535620878e-07, + "loss": 0.4086, + "num_tokens": 1470150269.0, + "reward": 1.81982421875, + "reward_std": 0.5035111904144287, + "rewards/accuracy_reward/mean": 0.03515625, + "rewards/accuracy_reward/std": 0.1843547374010086, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.91943359375, + "rewards/tag_count_reward/std": 0.2038721740245819, + "step": 2765 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2024.0, + "completions/mean_length": 853.833984375, + "completions/mean_terminated_length": 792.5318603515625, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "epoch": 0.944269010838952, + "grad_norm": 2.4273881912231445, + "kl": 6.5546875, + "learning_rate": 1.0856185165120433e-07, + "loss": 0.4685, + "num_tokens": 1470658808.0, + "reward": 1.88623046875, + "reward_std": 0.5840543508529663, + "rewards/accuracy_reward/mean": 0.10546875, + "rewards/accuracy_reward/std": 0.3074568510055542, + "rewards/format_reward/mean": 0.859375, + "rewards/format_reward/std": 0.3479743003845215, + "rewards/tag_count_reward/mean": 0.92138671875, + "rewards/tag_count_reward/std": 0.20343582332134247, + "step": 2766 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1927.0, + "completions/mean_length": 862.708984375, + "completions/mean_terminated_length": 822.0020751953125, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "epoch": 0.9446103951523428, + "grad_norm": 1.5315406322479248, + "kl": 6.375, + "learning_rate": 1.0845808448691141e-07, + "loss": 0.4185, + "num_tokens": 1471171075.0, + "reward": 1.84423828125, + "reward_std": 0.5585125684738159, + "rewards/accuracy_reward/mean": 0.06451612710952759, + "rewards/accuracy_reward/std": 0.2459181249141693, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.92041015625, + "rewards/tag_count_reward/std": 0.2030550092458725, + "step": 2767 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1995.0, + "completions/mean_length": 822.896484375, + "completions/mean_terminated_length": 754.6948852539062, + "completions/min_length": 177.0, + "completions/min_terminated_length": 177.0, + "epoch": 0.9449517794657335, + "grad_norm": 2.03684401512146, + "kl": 6.6171875, + "learning_rate": 1.0835494401060835e-07, + "loss": 0.4486, + "num_tokens": 1471670398.0, + "reward": 1.8759765625, + "reward_std": 0.5379860401153564, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.9326171875, + "rewards/tag_count_reward/std": 0.1911715567111969, + "step": 2768 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2013.0, + "completions/mean_length": 844.67578125, + "completions/mean_terminated_length": 818.2554931640625, + "completions/min_length": 164.0, + "completions/min_terminated_length": 164.0, + "epoch": 0.9452931637791243, + "grad_norm": 1.6256250143051147, + "kl": 5.1015625, + "learning_rate": 1.0825243036868424e-07, + "loss": 0.319, + "num_tokens": 1472174888.0, + "reward": 1.9091796875, + "reward_std": 0.5101285576820374, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.880859375, + "rewards/format_reward/std": 0.32427072525024414, + "rewards/tag_count_reward/mean": 0.9423828125, + "rewards/tag_count_reward/std": 0.1701863408088684, + "step": 2769 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 873.265625, + "completions/mean_terminated_length": 823.0224609375, + "completions/min_length": 112.0, + "completions/min_terminated_length": 112.0, + "epoch": 0.9456345480925151, + "grad_norm": 2.38149356842041, + "kl": 8.0859375, + "learning_rate": 1.081505437066386e-07, + "loss": 0.5097, + "num_tokens": 1472707488.0, + "reward": 1.7958984375, + "reward_std": 0.5217913389205933, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.9189453125, + "rewards/tag_count_reward/std": 0.2015654444694519, + "step": 2770 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.056640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 856.62109375, + "completions/mean_terminated_length": 785.0890502929688, + "completions/min_length": 157.0, + "completions/min_terminated_length": 157.0, + "epoch": 0.9459759324059059, + "grad_norm": 3.7725250720977783, + "kl": 9.3984375, + "learning_rate": 1.0804928416908073e-07, + "loss": 0.53, + "num_tokens": 1473224142.0, + "reward": 1.75146484375, + "reward_std": 0.6223288774490356, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.806640625, + "rewards/format_reward/std": 0.39531853795051575, + "rewards/tag_count_reward/mean": 0.89208984375, + "rewards/tag_count_reward/std": 0.24017061293125153, + "step": 2771 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.064453125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2002.0, + "completions/mean_length": 877.8359375, + "completions/mean_terminated_length": 797.21923828125, + "completions/min_length": 112.0, + "completions/min_terminated_length": 112.0, + "epoch": 0.9463173167192968, + "grad_norm": 5.367483139038086, + "kl": 9.8515625, + "learning_rate": 1.0794865189973011e-07, + "loss": 0.557, + "num_tokens": 1473763466.0, + "reward": 1.74267578125, + "reward_std": 0.5900819301605225, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.806640625, + "rewards/format_reward/std": 0.39531853795051575, + "rewards/tag_count_reward/mean": 0.89306640625, + "rewards/tag_count_reward/std": 0.23285804688930511, + "step": 2772 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 768.138671875, + "completions/mean_terminated_length": 713.3992309570312, + "completions/min_length": 135.0, + "completions/min_terminated_length": 135.0, + "epoch": 0.9466587010326876, + "grad_norm": 2.9520204067230225, + "kl": 9.2265625, + "learning_rate": 1.0784864704141585e-07, + "loss": 0.5612, + "num_tokens": 1474234529.0, + "reward": 1.9091796875, + "reward_std": 0.5921919941902161, + "rewards/accuracy_reward/mean": 0.119140625, + "rewards/accuracy_reward/std": 0.32427072525024414, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.9208984375, + "rewards/tag_count_reward/std": 0.20888477563858032, + "step": 2773 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 878.41015625, + "completions/mean_terminated_length": 840.681396484375, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "epoch": 0.9470000853460784, + "grad_norm": 3.5780584812164307, + "kl": 8.5078125, + "learning_rate": 1.0774926973607648e-07, + "loss": 0.4941, + "num_tokens": 1474761459.0, + "reward": 1.8134765625, + "reward_std": 0.5407140254974365, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.9228515625, + "rewards/tag_count_reward/std": 0.19821925461292267, + "step": 2774 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 843.189453125, + "completions/mean_terminated_length": 783.9364013671875, + "completions/min_length": 127.0, + "completions/min_terminated_length": 127.0, + "epoch": 0.9473414696594692, + "grad_norm": 2.634669542312622, + "kl": 9.1328125, + "learning_rate": 1.0765052012475998e-07, + "loss": 0.556, + "num_tokens": 1475264484.0, + "reward": 1.76513671875, + "reward_std": 0.5684515237808228, + "rewards/accuracy_reward/mean": 0.03125, + "rewards/accuracy_reward/std": 0.17416280508041382, + "rewards/format_reward/mean": 0.828125, + "rewards/format_reward/std": 0.3776407241821289, + "rewards/tag_count_reward/mean": 0.90576171875, + "rewards/tag_count_reward/std": 0.21791352331638336, + "step": 2775 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2030.0, + "completions/mean_length": 815.4453125, + "completions/mean_terminated_length": 760.1060791015625, + "completions/min_length": 112.0, + "completions/min_terminated_length": 112.0, + "epoch": 0.9476828539728599, + "grad_norm": 4.417301654815674, + "kl": 10.0625, + "learning_rate": 1.0755239834762326e-07, + "loss": 0.592, + "num_tokens": 1475757720.0, + "reward": 1.83447265625, + "reward_std": 0.5964944362640381, + "rewards/accuracy_reward/mean": 0.103515625, + "rewards/accuracy_reward/std": 0.30492907762527466, + "rewards/format_reward/mean": 0.818359375, + "rewards/format_reward/std": 0.38592514395713806, + "rewards/tag_count_reward/mean": 0.91259765625, + "rewards/tag_count_reward/std": 0.21054047346115112, + "step": 2776 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1997.0, + "completions/mean_length": 833.474609375, + "completions/mean_terminated_length": 789.2206420898438, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "epoch": 0.9480242382862507, + "grad_norm": 1.6135154962539673, + "kl": 7.328125, + "learning_rate": 1.0745490454393239e-07, + "loss": 0.44, + "num_tokens": 1476266875.0, + "reward": 1.798828125, + "reward_std": 0.5334725975990295, + "rewards/accuracy_reward/mean": 0.033203125, + "rewards/accuracy_reward/std": 0.17934183776378632, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.919921875, + "rewards/tag_count_reward/std": 0.19767135381698608, + "step": 2777 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2014.0, + "completions/mean_length": 828.712890625, + "completions/mean_terminated_length": 801.942138671875, + "completions/min_length": 75.0, + "completions/min_terminated_length": 75.0, + "epoch": 0.9483656225996415, + "grad_norm": 1.146044373512268, + "kl": 6.71875, + "learning_rate": 1.0735803885206191e-07, + "loss": 0.4044, + "num_tokens": 1476771064.0, + "reward": 1.87255859375, + "reward_std": 0.5109162330627441, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.876953125, + "rewards/format_reward/std": 0.32881227135658264, + "rewards/tag_count_reward/mean": 0.93896484375, + "rewards/tag_count_reward/std": 0.17711623013019562, + "step": 2778 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 828.203125, + "completions/mean_terminated_length": 778.6178588867188, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "epoch": 0.9487070069130323, + "grad_norm": 1.207013726234436, + "kl": 7.6796875, + "learning_rate": 1.0726180140949497e-07, + "loss": 0.4986, + "num_tokens": 1477281520.0, + "reward": 1.8486328125, + "reward_std": 0.5367643237113953, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.9287109375, + "rewards/tag_count_reward/std": 0.1979491263628006, + "step": 2779 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.06640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 868.1015625, + "completions/mean_terminated_length": 784.1757202148438, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "epoch": 0.9490483912264231, + "grad_norm": 1.251905083656311, + "kl": 7.5703125, + "learning_rate": 1.0716619235282295e-07, + "loss": 0.5291, + "num_tokens": 1477806500.0, + "reward": 1.8759765625, + "reward_std": 0.5983998775482178, + "rewards/accuracy_reward/mean": 0.119140625, + "rewards/accuracy_reward/std": 0.32427072525024414, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.9169921875, + "rewards/tag_count_reward/std": 0.21260276436805725, + "step": 2780 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2029.0, + "completions/mean_length": 833.78125, + "completions/mean_terminated_length": 794.6128540039062, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "epoch": 0.949389775539814, + "grad_norm": 1.1650217771530151, + "kl": 5.9921875, + "learning_rate": 1.0707121181774556e-07, + "loss": 0.3707, + "num_tokens": 1478317156.0, + "reward": 1.81884765625, + "reward_std": 0.5314297676086426, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.859375, + "rewards/format_reward/std": 0.3479743003845215, + "rewards/tag_count_reward/mean": 0.91845703125, + "rewards/tag_count_reward/std": 0.2117307335138321, + "step": 2781 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2039.0, + "completions/mean_length": 822.16015625, + "completions/mean_terminated_length": 772.3292236328125, + "completions/min_length": 122.0, + "completions/min_terminated_length": 122.0, + "epoch": 0.9497311598532048, + "grad_norm": 2.7264535427093506, + "kl": 6.0546875, + "learning_rate": 1.0697685993907009e-07, + "loss": 0.4079, + "num_tokens": 1478811750.0, + "reward": 1.833984375, + "reward_std": 0.5194734930992126, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.857421875, + "rewards/format_reward/std": 0.3499840497970581, + "rewards/tag_count_reward/mean": 0.92578125, + "rewards/tag_count_reward/std": 0.19686728715896606, + "step": 2782 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 776.794921875, + "completions/mean_terminated_length": 746.2860107421875, + "completions/min_length": 139.0, + "completions/min_terminated_length": 139.0, + "epoch": 0.9500725441665956, + "grad_norm": 1.6510119438171387, + "kl": 4.10546875, + "learning_rate": 1.0688313685071194e-07, + "loss": 0.2936, + "num_tokens": 1479280301.0, + "reward": 1.97412109375, + "reward_std": 0.44091886281967163, + "rewards/accuracy_reward/mean": 0.115234375, + "rewards/accuracy_reward/std": 0.3196168541908264, + "rewards/format_reward/mean": 0.904296875, + "rewards/format_reward/std": 0.2944713830947876, + "rewards/tag_count_reward/mean": 0.95458984375, + "rewards/tag_count_reward/std": 0.15722467005252838, + "step": 2783 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2028.0, + "completions/mean_length": 830.234375, + "completions/mean_terminated_length": 778.1507568359375, + "completions/min_length": 165.0, + "completions/min_terminated_length": 165.0, + "epoch": 0.9504139284799863, + "grad_norm": 2.9825966358184814, + "kl": 5.1796875, + "learning_rate": 1.067900426856939e-07, + "loss": 0.3634, + "num_tokens": 1479774613.0, + "reward": 1.88232421875, + "reward_std": 0.4768640995025635, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.88671875, + "rewards/format_reward/std": 0.3172462284564972, + "rewards/tag_count_reward/mean": 0.94287109375, + "rewards/tag_count_reward/std": 0.1749558448791504, + "step": 2784 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1948.0, + "completions/mean_length": 835.341796875, + "completions/mean_terminated_length": 786.0466918945312, + "completions/min_length": 39.0, + "completions/min_terminated_length": 39.0, + "epoch": 0.9507553127933771, + "grad_norm": 2.1468441486358643, + "kl": 4.765625, + "learning_rate": 1.0669757757614602e-07, + "loss": 0.3044, + "num_tokens": 1480288452.0, + "reward": 1.8671875, + "reward_std": 0.47573575377464294, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.880859375, + "rewards/format_reward/std": 0.32427072525024414, + "rewards/tag_count_reward/mean": 0.939453125, + "rewards/tag_count_reward/std": 0.1783159226179123, + "step": 2785 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2000.0, + "completions/mean_length": 796.875, + "completions/mean_terminated_length": 740.7020263671875, + "completions/min_length": 121.0, + "completions/min_terminated_length": 121.0, + "epoch": 0.9510966971067679, + "grad_norm": 1.9844145774841309, + "kl": 5.6640625, + "learning_rate": 1.0660574165330567e-07, + "loss": 0.399, + "num_tokens": 1480773092.0, + "reward": 1.94384765625, + "reward_std": 0.48710376024246216, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.908203125, + "rewards/format_reward/std": 0.289021372795105, + "rewards/tag_count_reward/mean": 0.94580078125, + "rewards/tag_count_reward/std": 0.18136507272720337, + "step": 2786 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2018.0, + "completions/mean_length": 821.798828125, + "completions/mean_terminated_length": 779.6868896484375, + "completions/min_length": 77.0, + "completions/min_terminated_length": 77.0, + "epoch": 0.9514380814201587, + "grad_norm": 1.1855788230895996, + "kl": 5.01953125, + "learning_rate": 1.065145350475171e-07, + "loss": 0.3068, + "num_tokens": 1481271917.0, + "reward": 1.93017578125, + "reward_std": 0.506824254989624, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.890625, + "rewards/format_reward/std": 0.31241437792778015, + "rewards/tag_count_reward/mean": 0.94189453125, + "rewards/tag_count_reward/std": 0.17463330924510956, + "step": 2787 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 851.8984375, + "completions/mean_terminated_length": 815.7987670898438, + "completions/min_length": 139.0, + "completions/min_terminated_length": 139.0, + "epoch": 0.9517794657335495, + "grad_norm": 1.280172348022461, + "kl": 4.97265625, + "learning_rate": 1.0642395788823144e-07, + "loss": 0.3283, + "num_tokens": 1481787961.0, + "reward": 1.896484375, + "reward_std": 0.4289102554321289, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.900390625, + "rewards/format_reward/std": 0.29977133870124817, + "rewards/tag_count_reward/mean": 0.9453125, + "rewards/tag_count_reward/std": 0.1781550794839859, + "step": 2788 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1926.0, + "completions/mean_length": 798.341796875, + "completions/mean_terminated_length": 747.5426635742188, + "completions/min_length": 47.0, + "completions/min_terminated_length": 47.0, + "epoch": 0.9521208500469404, + "grad_norm": 1.7842223644256592, + "kl": 6.50390625, + "learning_rate": 1.0633401030400637e-07, + "loss": 0.4106, + "num_tokens": 1482275576.0, + "reward": 1.89013671875, + "reward_std": 0.4526803493499756, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.88671875, + "rewards/format_reward/std": 0.3172462284564972, + "rewards/tag_count_reward/mean": 0.94482421875, + "rewards/tag_count_reward/std": 0.17277398705482483, + "step": 2789 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 828.2421875, + "completions/mean_terminated_length": 778.6585083007812, + "completions/min_length": 44.0, + "completions/min_terminated_length": 44.0, + "epoch": 0.9524622343603312, + "grad_norm": 1.977900743484497, + "kl": 5.6953125, + "learning_rate": 1.0624469242250607e-07, + "loss": 0.3367, + "num_tokens": 1482786180.0, + "reward": 1.90087890625, + "reward_std": 0.49716493487358093, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.93603515625, + "rewards/tag_count_reward/std": 0.18154938519001007, + "step": 2790 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1967.0, + "completions/mean_length": 773.541015625, + "completions/mean_terminated_length": 742.9540405273438, + "completions/min_length": 36.0, + "completions/min_terminated_length": 36.0, + "epoch": 0.952803618673722, + "grad_norm": 1.6823440790176392, + "kl": 5.61328125, + "learning_rate": 1.0615600437050094e-07, + "loss": 0.329, + "num_tokens": 1483263497.0, + "reward": 1.96923828125, + "reward_std": 0.4446501135826111, + "rewards/accuracy_reward/mean": 0.109375, + "rewards/accuracy_reward/std": 0.31241437792778015, + "rewards/format_reward/mean": 0.908203125, + "rewards/format_reward/std": 0.289021372795105, + "rewards/tag_count_reward/mean": 0.95166015625, + "rewards/tag_count_reward/std": 0.15944552421569824, + "step": 2791 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 799.955078125, + "completions/mean_terminated_length": 772.5529174804688, + "completions/min_length": 103.0, + "completions/min_terminated_length": 103.0, + "epoch": 0.9531450029871127, + "grad_norm": 1.415523886680603, + "kl": 6.4765625, + "learning_rate": 1.0606794627386739e-07, + "loss": 0.4152, + "num_tokens": 1483751570.0, + "reward": 1.859375, + "reward_std": 0.4958192706108093, + "rewards/accuracy_reward/mean": 0.05645161122083664, + "rewards/accuracy_reward/std": 0.23102475702762604, + "rewards/format_reward/mean": 0.8671875, + "rewards/format_reward/std": 0.33970388770103455, + "rewards/tag_count_reward/mean": 0.9375, + "rewards/tag_count_reward/std": 0.1843961924314499, + "step": 2792 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1972.0, + "completions/mean_length": 841.2734375, + "completions/mean_terminated_length": 781.9262084960938, + "completions/min_length": 23.0, + "completions/min_terminated_length": 23.0, + "epoch": 0.9534863873005035, + "grad_norm": 3.0615651607513428, + "kl": 8.5546875, + "learning_rate": 1.0598051825758785e-07, + "loss": 0.5124, + "num_tokens": 1484260062.0, + "reward": 1.859375, + "reward_std": 0.5691763162612915, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.8515625, + "rewards/format_reward/std": 0.35588082671165466, + "rewards/tag_count_reward/mean": 0.921875, + "rewards/tag_count_reward/std": 0.2021169811487198, + "step": 2793 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1917.0, + "completions/mean_length": 836.037109375, + "completions/mean_terminated_length": 789.3285522460938, + "completions/min_length": 135.0, + "completions/min_terminated_length": 135.0, + "epoch": 0.9538277716138943, + "grad_norm": 1.6534143686294556, + "kl": 6.7578125, + "learning_rate": 1.0589372044575035e-07, + "loss": 0.4172, + "num_tokens": 1484767281.0, + "reward": 1.8564453125, + "reward_std": 0.47521013021469116, + "rewards/accuracy_reward/mean": 0.038306452333927155, + "rewards/accuracy_reward/std": 0.19212883710861206, + "rewards/format_reward/mean": 0.8828125, + "rewards/format_reward/std": 0.32195815443992615, + "rewards/tag_count_reward/mean": 0.9365234375, + "rewards/tag_count_reward/std": 0.18472495675086975, + "step": 2794 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2004.0, + "completions/mean_length": 831.046875, + "completions/mean_terminated_length": 789.2525634765625, + "completions/min_length": 82.0, + "completions/min_terminated_length": 82.0, + "epoch": 0.9541691559272851, + "grad_norm": 2.2055423259735107, + "kl": 7.1328125, + "learning_rate": 1.0580755296154857e-07, + "loss": 0.4278, + "num_tokens": 1485270873.0, + "reward": 1.8720703125, + "reward_std": 0.4822779595851898, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.873046875, + "rewards/format_reward/std": 0.33324605226516724, + "rewards/tag_count_reward/mean": 0.9384765625, + "rewards/tag_count_reward/std": 0.18272781372070312, + "step": 2795 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2036.0, + "completions/mean_length": 825.9765625, + "completions/mean_terminated_length": 791.6224365234375, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "epoch": 0.9545105402406759, + "grad_norm": 2.20977783203125, + "kl": 7.453125, + "learning_rate": 1.0572201592728136e-07, + "loss": 0.4529, + "num_tokens": 1485766477.0, + "reward": 1.8544921875, + "reward_std": 0.5384957194328308, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.86328125, + "rewards/format_reward/std": 0.3438861668109894, + "rewards/tag_count_reward/mean": 0.9326171875, + "rewards/tag_count_reward/std": 0.1813209503889084, + "step": 2796 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2020.0, + "completions/mean_length": 833.939453125, + "completions/mean_terminated_length": 794.7761840820312, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "epoch": 0.9548519245540668, + "grad_norm": 1.5561234951019287, + "kl": 5.984375, + "learning_rate": 1.0563710946435309e-07, + "loss": 0.3802, + "num_tokens": 1486279038.0, + "reward": 1.88671875, + "reward_std": 0.48291343450546265, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.880859375, + "rewards/format_reward/std": 0.32427072525024414, + "rewards/tag_count_reward/mean": 0.94140625, + "rewards/tag_count_reward/std": 0.17270830273628235, + "step": 2797 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1973.0, + "completions/mean_length": 846.279296875, + "completions/mean_terminated_length": 805.0081176757812, + "completions/min_length": 120.0, + "completions/min_terminated_length": 120.0, + "epoch": 0.9551933088674576, + "grad_norm": 1.6758620738983154, + "kl": 5.45703125, + "learning_rate": 1.0555283369327283e-07, + "loss": 0.3251, + "num_tokens": 1486784653.0, + "reward": 1.84228515625, + "reward_std": 0.43197542428970337, + "rewards/accuracy_reward/mean": 0.013671875, + "rewards/accuracy_reward/std": 0.1162383034825325, + "rewards/format_reward/mean": 0.888671875, + "rewards/format_reward/std": 0.31484565138816833, + "rewards/tag_count_reward/mean": 0.93994140625, + "rewards/tag_count_reward/std": 0.18018634617328644, + "step": 2798 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2022.0, + "completions/mean_length": 777.490234375, + "completions/mean_terminated_length": 746.998046875, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "epoch": 0.9555346931808484, + "grad_norm": 2.062755584716797, + "kl": 3.61328125, + "learning_rate": 1.0546918873365457e-07, + "loss": 0.2445, + "num_tokens": 1487258040.0, + "reward": 1.96044921875, + "reward_std": 0.4145369529724121, + "rewards/accuracy_reward/mean": 0.09072580933570862, + "rewards/accuracy_reward/std": 0.2875087857246399, + "rewards/format_reward/mean": 0.9140625, + "rewards/format_reward/std": 0.28054583072662354, + "rewards/tag_count_reward/mean": 0.95849609375, + "rewards/tag_count_reward/std": 0.1445726603269577, + "step": 2799 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1979.0, + "completions/mean_length": 844.419921875, + "completions/mean_terminated_length": 810.5842895507812, + "completions/min_length": 125.0, + "completions/min_terminated_length": 125.0, + "epoch": 0.9558760774942391, + "grad_norm": 1.3885384798049927, + "kl": 4.453125, + "learning_rate": 1.0538617470421715e-07, + "loss": 0.3121, + "num_tokens": 1487764111.0, + "reward": 1.9013671875, + "reward_std": 0.4131018817424774, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.90625, + "rewards/format_reward/std": 0.29176566004753113, + "rewards/tag_count_reward/mean": 0.9521484375, + "rewards/tag_count_reward/std": 0.16149762272834778, + "step": 2800 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1953.0, + "completions/mean_length": 865.474609375, + "completions/mean_terminated_length": 827.32861328125, + "completions/min_length": 130.0, + "completions/min_terminated_length": 130.0, + "epoch": 0.9562174618076299, + "grad_norm": 1.4190959930419922, + "kl": 4.97265625, + "learning_rate": 1.0530379172278375e-07, + "loss": 0.3583, + "num_tokens": 1488289714.0, + "reward": 1.86181640625, + "reward_std": 0.46865037083625793, + "rewards/accuracy_reward/mean": 0.033203125, + "rewards/accuracy_reward/std": 0.17934183776378632, + "rewards/format_reward/mean": 0.884765625, + "rewards/format_reward/std": 0.3196168541908264, + "rewards/tag_count_reward/mean": 0.94384765625, + "rewards/tag_count_reward/std": 0.17031753063201904, + "step": 2801 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1994.0, + "completions/mean_length": 831.384765625, + "completions/mean_terminated_length": 802.18603515625, + "completions/min_length": 139.0, + "completions/min_terminated_length": 139.0, + "epoch": 0.9565588461210207, + "grad_norm": 1.2509667873382568, + "kl": 4.17578125, + "learning_rate": 1.0522203990628196e-07, + "loss": 0.2749, + "num_tokens": 1488792359.0, + "reward": 1.9306640625, + "reward_std": 0.47656846046447754, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.892578125, + "rewards/format_reward/std": 0.30995169281959534, + "rewards/tag_count_reward/mean": 0.9443359375, + "rewards/tag_count_reward/std": 0.162770614027977, + "step": 2802 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1968.0, + "completions/mean_length": 777.349609375, + "completions/mean_terminated_length": 749.4511108398438, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "epoch": 0.9569002304344115, + "grad_norm": 1.0817357301712036, + "kl": 4.1328125, + "learning_rate": 1.0514091937074349e-07, + "loss": 0.2997, + "num_tokens": 1489267802.0, + "reward": 1.93310546875, + "reward_std": 0.48178669810295105, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.900390625, + "rewards/format_reward/std": 0.29977133870124817, + "rewards/tag_count_reward/mean": 0.93701171875, + "rewards/tag_count_reward/std": 0.18456129729747772, + "step": 2803 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 820.345703125, + "completions/mean_terminated_length": 798.3796997070312, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "epoch": 0.9572416147478023, + "grad_norm": 1.4881634712219238, + "kl": 4.26953125, + "learning_rate": 1.050604302313042e-07, + "loss": 0.2953, + "num_tokens": 1489766955.0, + "reward": 1.94677734375, + "reward_std": 0.4905741214752197, + "rewards/accuracy_reward/mean": 0.099609375, + "rewards/accuracy_reward/std": 0.29977133870124817, + "rewards/format_reward/mean": 0.896484375, + "rewards/format_reward/std": 0.30492907762527466, + "rewards/tag_count_reward/mean": 0.95068359375, + "rewards/tag_count_reward/std": 0.15837518870830536, + "step": 2804 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1871.0, + "completions/mean_length": 743.041015625, + "completions/mean_terminated_length": 727.5671997070312, + "completions/min_length": 130.0, + "completions/min_terminated_length": 130.0, + "epoch": 0.9575829990611932, + "grad_norm": 1.4065176248550415, + "kl": 3.58203125, + "learning_rate": 1.0498057260220361e-07, + "loss": 0.2319, + "num_tokens": 1490223936.0, + "reward": 1.9267578125, + "reward_std": 0.4127148687839508, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.91015625, + "rewards/format_reward/std": 0.2862374484539032, + "rewards/tag_count_reward/mean": 0.9560546875, + "rewards/tag_count_reward/std": 0.14511774480342865, + "step": 2805 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 876.193359375, + "completions/mean_terminated_length": 845.6653442382812, + "completions/min_length": 129.0, + "completions/min_terminated_length": 129.0, + "epoch": 0.957924383374584, + "grad_norm": 1.7111616134643555, + "kl": 6.0234375, + "learning_rate": 1.0490134659678501e-07, + "loss": 0.4275, + "num_tokens": 1490752083.0, + "reward": 1.85595703125, + "reward_std": 0.5251683592796326, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.87890625, + "rewards/format_reward/std": 0.3265552520751953, + "rewards/tag_count_reward/mean": 0.93017578125, + "rewards/tag_count_reward/std": 0.1918918341398239, + "step": 2806 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1978.0, + "completions/mean_length": 800.888671875, + "completions/mean_terminated_length": 776.0458374023438, + "completions/min_length": 120.0, + "completions/min_terminated_length": 120.0, + "epoch": 0.9582657676879748, + "grad_norm": 1.5385019779205322, + "kl": 4.2265625, + "learning_rate": 1.0482275232749527e-07, + "loss": 0.2499, + "num_tokens": 1491240234.0, + "reward": 1.95068359375, + "reward_std": 0.39756447076797485, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.9140625, + "rewards/format_reward/std": 0.28054583072662354, + "rewards/tag_count_reward/mean": 0.95849609375, + "rewards/tag_count_reward/std": 0.1445726603269577, + "step": 2807 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2014.0, + "completions/mean_length": 813.478515625, + "completions/mean_terminated_length": 783.8500366210938, + "completions/min_length": 140.0, + "completions/min_terminated_length": 140.0, + "epoch": 0.9586071520013656, + "grad_norm": 1.1500355005264282, + "kl": 5.3203125, + "learning_rate": 1.0474478990588456e-07, + "loss": 0.3168, + "num_tokens": 1491737615.0, + "reward": 1.85009765625, + "reward_std": 0.4937742054462433, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.92822265625, + "rewards/tag_count_reward/std": 0.1898845136165619, + "step": 2808 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.009765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 741.298828125, + "completions/mean_terminated_length": 728.4122314453125, + "completions/min_length": 117.0, + "completions/min_terminated_length": 117.0, + "epoch": 0.9589485363147563, + "grad_norm": 1.122118592262268, + "kl": 3.59765625, + "learning_rate": 1.0466745944260631e-07, + "loss": 0.2334, + "num_tokens": 1492194920.0, + "reward": 1.87744140625, + "reward_std": 0.4407830834388733, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.8828125, + "rewards/format_reward/std": 0.32195815443992615, + "rewards/tag_count_reward/mean": 0.94970703125, + "rewards/tag_count_reward/std": 0.15175074338912964, + "step": 2809 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1985.0, + "completions/mean_length": 879.783203125, + "completions/mean_terminated_length": 846.9417114257812, + "completions/min_length": 141.0, + "completions/min_terminated_length": 141.0, + "epoch": 0.9592899206281471, + "grad_norm": 1.481045126914978, + "kl": 5.234375, + "learning_rate": 1.0459076104741699e-07, + "loss": 0.3218, + "num_tokens": 1492723961.0, + "reward": 1.86279296875, + "reward_std": 0.5158882141113281, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.875, + "rewards/format_reward/std": 0.3310423493385315, + "rewards/tag_count_reward/mean": 0.93505859375, + "rewards/tag_count_reward/std": 0.1777946949005127, + "step": 2810 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2025.0, + "completions/mean_length": 777.560546875, + "completions/mean_terminated_length": 752.2529907226562, + "completions/min_length": 117.0, + "completions/min_terminated_length": 117.0, + "epoch": 0.9596313049415379, + "grad_norm": 1.2403146028518677, + "kl": 4.7578125, + "learning_rate": 1.045146948291758e-07, + "loss": 0.32, + "num_tokens": 1493210216.0, + "reward": 1.95751953125, + "reward_std": 0.5043487548828125, + "rewards/accuracy_reward/mean": 0.125, + "rewards/accuracy_reward/std": 0.3310423493385315, + "rewards/format_reward/mean": 0.888671875, + "rewards/format_reward/std": 0.31484565138816833, + "rewards/tag_count_reward/mean": 0.94384765625, + "rewards/tag_count_reward/std": 0.15763740241527557, + "step": 2811 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2008.0, + "completions/mean_length": 837.96875, + "completions/mean_terminated_length": 781.05517578125, + "completions/min_length": 98.0, + "completions/min_terminated_length": 98.0, + "epoch": 0.9599726892549287, + "grad_norm": 1.8360158205032349, + "kl": 7.0390625, + "learning_rate": 1.0443926089584498e-07, + "loss": 0.4611, + "num_tokens": 1493719512.0, + "reward": 1.84912109375, + "reward_std": 0.532717227935791, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.876953125, + "rewards/format_reward/std": 0.32881227135658264, + "rewards/tag_count_reward/mean": 0.92919921875, + "rewards/tag_count_reward/std": 0.1996619999408722, + "step": 2812 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2011.0, + "completions/mean_length": 775.24609375, + "completions/mean_terminated_length": 752.47314453125, + "completions/min_length": 51.0, + "completions/min_terminated_length": 51.0, + "epoch": 0.9603140735683195, + "grad_norm": 1.177033543586731, + "kl": 5.24609375, + "learning_rate": 1.0436445935448916e-07, + "loss": 0.3232, + "num_tokens": 1494202086.0, + "reward": 1.89501953125, + "reward_std": 0.5035998821258545, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.87890625, + "rewards/format_reward/std": 0.3265552520751953, + "rewards/tag_count_reward/mean": 0.94189453125, + "rewards/tag_count_reward/std": 0.17602849006652832, + "step": 2813 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2046.0, + "completions/mean_length": 768.931640625, + "completions/mean_terminated_length": 748.6290283203125, + "completions/min_length": 124.0, + "completions/min_terminated_length": 124.0, + "epoch": 0.9606554578817104, + "grad_norm": 1.8969199657440186, + "kl": 4.5859375, + "learning_rate": 1.0429029031127539e-07, + "loss": 0.3024, + "num_tokens": 1494671603.0, + "reward": 1.943359375, + "reward_std": 0.48436421155929565, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.908203125, + "rewards/format_reward/std": 0.289021372795105, + "rewards/tag_count_reward/mean": 0.951171875, + "rewards/tag_count_reward/std": 0.16421131789684296, + "step": 2814 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2041.0, + "completions/mean_length": 727.548828125, + "completions/mean_terminated_length": 690.4276733398438, + "completions/min_length": 94.0, + "completions/min_terminated_length": 94.0, + "epoch": 0.9609968421951012, + "grad_norm": 1.0462123155593872, + "kl": 6.5234375, + "learning_rate": 1.042167538714731e-07, + "loss": 0.4106, + "num_tokens": 1495124076.0, + "reward": 1.837890625, + "reward_std": 0.4597313106060028, + "rewards/accuracy_reward/mean": 0.02734375, + "rewards/accuracy_reward/std": 0.16324250400066376, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.939453125, + "rewards/tag_count_reward/std": 0.17131954431533813, + "step": 2815 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2005.0, + "completions/mean_length": 752.203125, + "completions/mean_terminated_length": 713.0945434570312, + "completions/min_length": 109.0, + "completions/min_terminated_length": 109.0, + "epoch": 0.961338226508492, + "grad_norm": 2.0134124755859375, + "kl": 5.66015625, + "learning_rate": 1.0414385013945384e-07, + "loss": 0.3713, + "num_tokens": 1495589060.0, + "reward": 1.89794921875, + "reward_std": 0.4629552364349365, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.89453125, + "rewards/format_reward/std": 0.3074568510055542, + "rewards/tag_count_reward/mean": 0.94482421875, + "rewards/tag_count_reward/std": 0.16919739544391632, + "step": 2816 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2015.0, + "completions/mean_length": 740.59375, + "completions/mean_terminated_length": 722.4713134765625, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.9616796108218827, + "grad_norm": 0.7559698820114136, + "kl": 4.0234375, + "learning_rate": 1.040715792186911e-07, + "loss": 0.2399, + "num_tokens": 1496041124.0, + "reward": 1.958984375, + "reward_std": 0.3937632739543915, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.9296875, + "rewards/format_reward/std": 0.25592297315597534, + "rewards/tag_count_reward/mean": 0.95703125, + "rewards/tag_count_reward/std": 0.15438589453697205, + "step": 2817 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2042.0, + "completions/mean_length": 772.58984375, + "completions/mean_terminated_length": 723.4360961914062, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "epoch": 0.9620209951352735, + "grad_norm": 1.7330492734909058, + "kl": 7.6171875, + "learning_rate": 1.0399994121176025e-07, + "loss": 0.5064, + "num_tokens": 1496510226.0, + "reward": 1.86181640625, + "reward_std": 0.5165878534317017, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.87890625, + "rewards/format_reward/std": 0.3265552520751953, + "rewards/tag_count_reward/mean": 0.92822265625, + "rewards/tag_count_reward/std": 0.19931232929229736, + "step": 2818 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2002.0, + "completions/mean_length": 731.984375, + "completions/mean_terminated_length": 708.4373168945312, + "completions/min_length": 135.0, + "completions/min_terminated_length": 135.0, + "epoch": 0.9623623794486643, + "grad_norm": 0.916216254234314, + "kl": 5.51953125, + "learning_rate": 1.039289362203383e-07, + "loss": 0.3396, + "num_tokens": 1496957050.0, + "reward": 1.912109375, + "reward_std": 0.4761887192726135, + "rewards/accuracy_reward/mean": 0.06653226166963577, + "rewards/accuracy_reward/std": 0.2494617998600006, + "rewards/format_reward/mean": 0.8984375, + "rewards/format_reward/std": 0.30236753821372986, + "rewards/tag_count_reward/mean": 0.94921875, + "rewards/tag_count_reward/std": 0.16361670196056366, + "step": 2819 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2014.0, + "completions/mean_length": 743.6640625, + "completions/mean_terminated_length": 709.683349609375, + "completions/min_length": 98.0, + "completions/min_terminated_length": 98.0, + "epoch": 0.9627037637620551, + "grad_norm": 1.6667364835739136, + "kl": 6.5703125, + "learning_rate": 1.0385856434520387e-07, + "loss": 0.3979, + "num_tokens": 1497408062.0, + "reward": 1.9189453125, + "reward_std": 0.4979446232318878, + "rewards/accuracy_reward/mean": 0.083984375, + "rewards/accuracy_reward/std": 0.2776356339454651, + "rewards/format_reward/mean": 0.896484375, + "rewards/format_reward/std": 0.30492907762527466, + "rewards/tag_count_reward/mean": 0.9384765625, + "rewards/tag_count_reward/std": 0.18406164646148682, + "step": 2820 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1958.0, + "completions/mean_length": 714.109375, + "completions/mean_terminated_length": 682.0960083007812, + "completions/min_length": 102.0, + "completions/min_terminated_length": 102.0, + "epoch": 0.963045148075446, + "grad_norm": 2.4002110958099365, + "kl": 5.39453125, + "learning_rate": 1.0378882568623697e-07, + "loss": 0.2916, + "num_tokens": 1497846518.0, + "reward": 1.927734375, + "reward_std": 0.47919708490371704, + "rewards/accuracy_reward/mean": 0.103515625, + "rewards/accuracy_reward/std": 0.30492907762527466, + "rewards/format_reward/mean": 0.88671875, + "rewards/format_reward/std": 0.3172462284564972, + "rewards/tag_count_reward/mean": 0.9375, + "rewards/tag_count_reward/std": 0.17625701427459717, + "step": 2821 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2012.0, + "completions/mean_length": 740.388671875, + "completions/mean_terminated_length": 709.0060424804688, + "completions/min_length": 146.0, + "completions/min_terminated_length": 146.0, + "epoch": 0.9633865323888368, + "grad_norm": 1.3741546869277954, + "kl": 6.1640625, + "learning_rate": 1.037197203424189e-07, + "loss": 0.4227, + "num_tokens": 1498310253.0, + "reward": 1.91650390625, + "reward_std": 0.4642665982246399, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.90234375, + "rewards/format_reward/std": 0.29713961482048035, + "rewards/tag_count_reward/mean": 0.94970703125, + "rewards/tag_count_reward/std": 0.16188986599445343, + "step": 2822 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05859375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 841.515625, + "completions/mean_terminated_length": 766.4232788085938, + "completions/min_length": 67.0, + "completions/min_terminated_length": 67.0, + "epoch": 0.9637279167022276, + "grad_norm": 3.3093695640563965, + "kl": 9.9765625, + "learning_rate": 1.0365124841183199e-07, + "loss": 0.6156, + "num_tokens": 1498815157.0, + "reward": 1.765625, + "reward_std": 0.5490684509277344, + "rewards/accuracy_reward/mean": 0.03125, + "rewards/accuracy_reward/std": 0.17416280508041382, + "rewards/format_reward/mean": 0.828125, + "rewards/format_reward/std": 0.3776407241821289, + "rewards/tag_count_reward/mean": 0.90625, + "rewards/tag_count_reward/std": 0.2178439050912857, + "step": 2823 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1923.0, + "completions/mean_length": 770.0625, + "completions/mean_terminated_length": 734.1365356445312, + "completions/min_length": 102.0, + "completions/min_terminated_length": 102.0, + "epoch": 0.9640693010156184, + "grad_norm": 2.1383633613586426, + "kl": 6.3671875, + "learning_rate": 1.0358340999165966e-07, + "loss": 0.4144, + "num_tokens": 1499294293.0, + "reward": 1.8857421875, + "reward_std": 0.5147566795349121, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.9384765625, + "rewards/tag_count_reward/std": 0.17451083660125732, + "step": 2824 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2034.0, + "completions/mean_length": 742.251953125, + "completions/mean_terminated_length": 700.1310424804688, + "completions/min_length": 88.0, + "completions/min_terminated_length": 88.0, + "epoch": 0.9644106853290091, + "grad_norm": 1.4135212898254395, + "kl": 6.84375, + "learning_rate": 1.035162051781861e-07, + "loss": 0.4496, + "num_tokens": 1499755478.0, + "reward": 1.8916015625, + "reward_std": 0.491312712430954, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.88671875, + "rewards/format_reward/std": 0.3172462284564972, + "rewards/tag_count_reward/mean": 0.9404296875, + "rewards/tag_count_reward/std": 0.17864517867565155, + "step": 2825 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1872.0, + "completions/mean_length": 737.171875, + "completions/mean_terminated_length": 708.3912353515625, + "completions/min_length": 130.0, + "completions/min_terminated_length": 130.0, + "epoch": 0.9647520696423999, + "grad_norm": 1.6564372777938843, + "kl": 6.53125, + "learning_rate": 1.0344963406679633e-07, + "loss": 0.4448, + "num_tokens": 1500214430.0, + "reward": 1.90380859375, + "reward_std": 0.4616513252258301, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.888671875, + "rewards/format_reward/std": 0.31484565138816833, + "rewards/tag_count_reward/mean": 0.94677734375, + "rewards/tag_count_reward/std": 0.16544538736343384, + "step": 2826 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1922.0, + "completions/mean_length": 758.19140625, + "completions/mean_terminated_length": 708.4827270507812, + "completions/min_length": 109.0, + "completions/min_terminated_length": 109.0, + "epoch": 0.9650934539557907, + "grad_norm": 1.8987891674041748, + "kl": 6.40234375, + "learning_rate": 1.0338369675197584e-07, + "loss": 0.4253, + "num_tokens": 1500675296.0, + "reward": 1.83447265625, + "reward_std": 0.5152586102485657, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.857421875, + "rewards/format_reward/std": 0.3499840497970581, + "rewards/tag_count_reward/mean": 0.92822265625, + "rewards/tag_count_reward/std": 0.1911684274673462, + "step": 2827 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1682.0, + "completions/mean_length": 707.9921875, + "completions/mean_terminated_length": 670.3212890625, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "epoch": 0.9654348382691815, + "grad_norm": 1.7056914567947388, + "kl": 6.8984375, + "learning_rate": 1.0331839332731053e-07, + "loss": 0.4827, + "num_tokens": 1501127884.0, + "reward": 1.8330078125, + "reward_std": 0.5483074188232422, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.9208984375, + "rewards/tag_count_reward/std": 0.20593629777431488, + "step": 2828 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2019.0, + "completions/mean_length": 705.625, + "completions/mean_terminated_length": 673.4080200195312, + "completions/min_length": 116.0, + "completions/min_terminated_length": 116.0, + "epoch": 0.9657762225825723, + "grad_norm": 2.8771121501922607, + "kl": 6.9296875, + "learning_rate": 1.0325372388548673e-07, + "loss": 0.4234, + "num_tokens": 1501569340.0, + "reward": 1.90087890625, + "reward_std": 0.5529721975326538, + "rewards/accuracy_reward/mean": 0.111328125, + "rewards/accuracy_reward/std": 0.31484565138816833, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.92822265625, + "rewards/tag_count_reward/std": 0.19808122515678406, + "step": 2829 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1982.0, + "completions/mean_length": 727.29296875, + "completions/mean_terminated_length": 708.9861450195312, + "completions/min_length": 153.0, + "completions/min_terminated_length": 153.0, + "epoch": 0.9661176068959632, + "grad_norm": 1.8524610996246338, + "kl": 5.8359375, + "learning_rate": 1.0318968851829084e-07, + "loss": 0.3779, + "num_tokens": 1502022946.0, + "reward": 1.89208984375, + "reward_std": 0.4285368025302887, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.888671875, + "rewards/format_reward/std": 0.31484565138816833, + "rewards/tag_count_reward/mean": 0.94482421875, + "rewards/tag_count_reward/std": 0.17348045110702515, + "step": 2830 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1998.0, + "completions/mean_length": 768.84765625, + "completions/mean_terminated_length": 714.1385498046875, + "completions/min_length": 108.0, + "completions/min_terminated_length": 108.0, + "epoch": 0.966458991209354, + "grad_norm": 2.203659772872925, + "kl": 8.4375, + "learning_rate": 1.031262873166094e-07, + "loss": 0.5622, + "num_tokens": 1502492548.0, + "reward": 1.84619140625, + "reward_std": 0.5414794683456421, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.91650390625, + "rewards/tag_count_reward/std": 0.2056826800107956, + "step": 2831 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1998.0, + "completions/mean_length": 744.240234375, + "completions/mean_terminated_length": 704.8912963867188, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "epoch": 0.9668003755227448, + "grad_norm": 1.5507378578186035, + "kl": 6.28515625, + "learning_rate": 1.0306352037042878e-07, + "loss": 0.4382, + "num_tokens": 1502951487.0, + "reward": 1.8828125, + "reward_std": 0.5205909013748169, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.880859375, + "rewards/format_reward/std": 0.32427072525024414, + "rewards/tag_count_reward/mean": 0.931640625, + "rewards/tag_count_reward/std": 0.19337067008018494, + "step": 2832 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2041.0, + "completions/mean_length": 771.90234375, + "completions/mean_terminated_length": 733.3883056640625, + "completions/min_length": 66.0, + "completions/min_terminated_length": 66.0, + "epoch": 0.9671417598361355, + "grad_norm": 2.5833253860473633, + "kl": 6.265625, + "learning_rate": 1.0300138776883521e-07, + "loss": 0.4523, + "num_tokens": 1503416205.0, + "reward": 1.85107421875, + "reward_std": 0.49749863147735596, + "rewards/accuracy_reward/mean": 0.037109375, + "rewards/accuracy_reward/std": 0.18921469151973724, + "rewards/format_reward/mean": 0.8828125, + "rewards/format_reward/std": 0.32195815443992615, + "rewards/tag_count_reward/mean": 0.93115234375, + "rewards/tag_count_reward/std": 0.1909683793783188, + "step": 2833 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1977.0, + "completions/mean_length": 780.7109375, + "completions/mean_terminated_length": 745.0842895507812, + "completions/min_length": 31.0, + "completions/min_terminated_length": 31.0, + "epoch": 0.9674831441495263, + "grad_norm": 1.1714969873428345, + "kl": 7.078125, + "learning_rate": 1.0293988960001453e-07, + "loss": 0.4828, + "num_tokens": 1503891801.0, + "reward": 1.8349609375, + "reward_std": 0.531904399394989, + "rewards/accuracy_reward/mean": 0.041015625, + "rewards/accuracy_reward/std": 0.19852031767368317, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.9248046875, + "rewards/tag_count_reward/std": 0.19958361983299255, + "step": 2834 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 825.32421875, + "completions/mean_terminated_length": 762.5585327148438, + "completions/min_length": 147.0, + "completions/min_terminated_length": 147.0, + "epoch": 0.9678245284629171, + "grad_norm": 1.9260815382003784, + "kl": 7.890625, + "learning_rate": 1.0287902595125212e-07, + "loss": 0.5501, + "num_tokens": 1504382223.0, + "reward": 1.787109375, + "reward_std": 0.5436063408851624, + "rewards/accuracy_reward/mean": 0.03125, + "rewards/accuracy_reward/std": 0.17416280508041382, + "rewards/format_reward/mean": 0.83984375, + "rewards/format_reward/std": 0.3671095669269562, + "rewards/tag_count_reward/mean": 0.916015625, + "rewards/tag_count_reward/std": 0.2087314873933792, + "step": 2835 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 784.662109375, + "completions/mean_terminated_length": 735.9736328125, + "completions/min_length": 110.0, + "completions/min_terminated_length": 110.0, + "epoch": 0.9681659127763079, + "grad_norm": 2.5262982845306396, + "kl": 6.96875, + "learning_rate": 1.0281879690893287e-07, + "loss": 0.4852, + "num_tokens": 1504863810.0, + "reward": 1.865234375, + "reward_std": 0.5836403369903564, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.86328125, + "rewards/format_reward/std": 0.3438861668109894, + "rewards/tag_count_reward/mean": 0.923828125, + "rewards/tag_count_reward/std": 0.2034645974636078, + "step": 2836 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2006.0, + "completions/mean_length": 814.23046875, + "completions/mean_terminated_length": 779.546142578125, + "completions/min_length": 152.0, + "completions/min_terminated_length": 152.0, + "epoch": 0.9685072970896987, + "grad_norm": 0.7660391926765442, + "kl": 6.9921875, + "learning_rate": 1.0275920255854082e-07, + "loss": 0.4373, + "num_tokens": 1505357368.0, + "reward": 1.85009765625, + "reward_std": 0.49968987703323364, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.859375, + "rewards/format_reward/std": 0.3479743003845215, + "rewards/tag_count_reward/mean": 0.93408203125, + "rewards/tag_count_reward/std": 0.18219561874866486, + "step": 2837 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1935.0, + "completions/mean_length": 787.3046875, + "completions/mean_terminated_length": 738.718017578125, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "epoch": 0.9688486814030896, + "grad_norm": 1.3993850946426392, + "kl": 6.453125, + "learning_rate": 1.0270024298465929e-07, + "loss": 0.4072, + "num_tokens": 1505843044.0, + "reward": 1.81982421875, + "reward_std": 0.5220248699188232, + "rewards/accuracy_reward/mean": 0.046875, + "rewards/accuracy_reward/std": 0.21157780289649963, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.92724609375, + "rewards/tag_count_reward/std": 0.19834154844284058, + "step": 2838 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1991.0, + "completions/mean_length": 802.654296875, + "completions/mean_terminated_length": 744.0797119140625, + "completions/min_length": 96.0, + "completions/min_terminated_length": 96.0, + "epoch": 0.9691900657164804, + "grad_norm": 1.2276525497436523, + "kl": 8.109375, + "learning_rate": 1.0264191827097057e-07, + "loss": 0.527, + "num_tokens": 1506330339.0, + "reward": 1.8525390625, + "reward_std": 0.5859047174453735, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.9169921875, + "rewards/tag_count_reward/std": 0.21028900146484375, + "step": 2839 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2028.0, + "completions/mean_length": 789.37109375, + "completions/mean_terminated_length": 738.207275390625, + "completions/min_length": 130.0, + "completions/min_terminated_length": 130.0, + "epoch": 0.9695314500298712, + "grad_norm": 1.2961440086364746, + "kl": 7.0703125, + "learning_rate": 1.0258422850025601e-07, + "loss": 0.4479, + "num_tokens": 1506817233.0, + "reward": 1.8408203125, + "reward_std": 0.48050573468208313, + "rewards/accuracy_reward/mean": 0.0390625, + "rewards/accuracy_reward/std": 0.1939331740140915, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.9326171875, + "rewards/tag_count_reward/std": 0.1879453957080841, + "step": 2840 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1958.0, + "completions/mean_length": 739.642578125, + "completions/mean_terminated_length": 702.8613891601562, + "completions/min_length": 96.0, + "completions/min_terminated_length": 96.0, + "epoch": 0.9698728343432619, + "grad_norm": 1.3730722665786743, + "kl": 6.4375, + "learning_rate": 1.025271737543956e-07, + "loss": 0.426, + "num_tokens": 1507276890.0, + "reward": 1.87060546875, + "reward_std": 0.4937661290168762, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.93701171875, + "rewards/tag_count_reward/std": 0.1812174916267395, + "step": 2841 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2017.0, + "completions/mean_length": 763.080078125, + "completions/mean_terminated_length": 729.605224609375, + "completions/min_length": 64.0, + "completions/min_terminated_length": 64.0, + "epoch": 0.9702142186566527, + "grad_norm": 1.189055323600769, + "kl": 6.7109375, + "learning_rate": 1.0247075411436815e-07, + "loss": 0.4136, + "num_tokens": 1507753651.0, + "reward": 1.82861328125, + "reward_std": 0.491859495639801, + "rewards/accuracy_reward/mean": 0.033203125, + "rewards/accuracy_reward/std": 0.17934183776378632, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.93017578125, + "rewards/tag_count_reward/std": 0.18932512402534485, + "step": 2842 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1948.0, + "completions/mean_length": 765.400390625, + "completions/mean_terminated_length": 726.6901245117188, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "epoch": 0.9705556029700435, + "grad_norm": 1.5975303649902344, + "kl": 6.0390625, + "learning_rate": 1.0241496966025103e-07, + "loss": 0.3615, + "num_tokens": 1508221024.0, + "reward": 1.91015625, + "reward_std": 0.5483856797218323, + "rewards/accuracy_reward/mean": 0.099609375, + "rewards/accuracy_reward/std": 0.29977133870124817, + "rewards/format_reward/mean": 0.876953125, + "rewards/format_reward/std": 0.32881227135658264, + "rewards/tag_count_reward/mean": 0.93359375, + "rewards/tag_count_reward/std": 0.19278669357299805, + "step": 2843 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1983.0, + "completions/mean_length": 786.294921875, + "completions/mean_terminated_length": 740.3218994140625, + "completions/min_length": 98.0, + "completions/min_terminated_length": 98.0, + "epoch": 0.9708969872834343, + "grad_norm": 1.0943127870559692, + "kl": 6.7578125, + "learning_rate": 1.0235982047121997e-07, + "loss": 0.4222, + "num_tokens": 1508703735.0, + "reward": 1.83642578125, + "reward_std": 0.5712246298789978, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.845703125, + "rewards/format_reward/std": 0.36158639192581177, + "rewards/tag_count_reward/mean": 0.91845703125, + "rewards/tag_count_reward/std": 0.20408298075199127, + "step": 2844 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2011.0, + "completions/mean_length": 739.03125, + "completions/mean_terminated_length": 718.2540283203125, + "completions/min_length": 129.0, + "completions/min_terminated_length": 129.0, + "epoch": 0.9712383715968251, + "grad_norm": 2.157069444656372, + "kl": 4.515625, + "learning_rate": 1.0230530662554937e-07, + "loss": 0.299, + "num_tokens": 1509163847.0, + "reward": 1.931640625, + "reward_std": 0.45786046981811523, + "rewards/accuracy_reward/mean": 0.078125, + "rewards/accuracy_reward/std": 0.26863065361976624, + "rewards/format_reward/mean": 0.904296875, + "rewards/format_reward/std": 0.2944713830947876, + "rewards/tag_count_reward/mean": 0.94921875, + "rewards/tag_count_reward/std": 0.16876834630966187, + "step": 2845 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1953.0, + "completions/mean_length": 725.2421875, + "completions/mean_terminated_length": 701.5745239257812, + "completions/min_length": 135.0, + "completions/min_terminated_length": 135.0, + "epoch": 0.971579755910216, + "grad_norm": 1.0849651098251343, + "kl": 5.2578125, + "learning_rate": 1.0225142820061143e-07, + "loss": 0.3072, + "num_tokens": 1509613459.0, + "reward": 1.875, + "reward_std": 0.43959301710128784, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.884765625, + "rewards/format_reward/std": 0.3196168541908264, + "rewards/tag_count_reward/mean": 0.9453125, + "rewards/tag_count_reward/std": 0.16754092276096344, + "step": 2846 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1993.0, + "completions/mean_length": 759.966796875, + "completions/mean_terminated_length": 710.3265380859375, + "completions/min_length": 130.0, + "completions/min_terminated_length": 130.0, + "epoch": 0.9719211402236068, + "grad_norm": 0.8330246806144714, + "kl": 7.390625, + "learning_rate": 1.0219818527287691e-07, + "loss": 0.4548, + "num_tokens": 1510080834.0, + "reward": 1.87548828125, + "reward_std": 0.5958338379859924, + "rewards/accuracy_reward/mean": 0.103515625, + "rewards/accuracy_reward/std": 0.30492907762527466, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.92431640625, + "rewards/tag_count_reward/std": 0.1997050791978836, + "step": 2847 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1832.0, + "completions/mean_length": 773.40234375, + "completions/mean_terminated_length": 732.2862548828125, + "completions/min_length": 65.0, + "completions/min_terminated_length": 65.0, + "epoch": 0.9722625245369976, + "grad_norm": 1.3064340353012085, + "kl": 7.0234375, + "learning_rate": 1.0214557791791425e-07, + "loss": 0.4567, + "num_tokens": 1510559184.0, + "reward": 1.90576171875, + "reward_std": 0.5227937698364258, + "rewards/accuracy_reward/mean": 0.1015625, + "rewards/accuracy_reward/std": 0.30236753821372986, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.93505859375, + "rewards/tag_count_reward/std": 0.18187542259693146, + "step": 2848 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1946.0, + "completions/mean_length": 811.189453125, + "completions/mean_terminated_length": 778.9679565429688, + "completions/min_length": 120.0, + "completions/min_terminated_length": 120.0, + "epoch": 0.9726039088503883, + "grad_norm": 1.6328932046890259, + "kl": 6.5234375, + "learning_rate": 1.0209360621039007e-07, + "loss": 0.4096, + "num_tokens": 1511060273.0, + "reward": 1.81884765625, + "reward_std": 0.533811092376709, + "rewards/accuracy_reward/mean": 0.04435483738780022, + "rewards/accuracy_reward/std": 0.2060900777578354, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.92822265625, + "rewards/tag_count_reward/std": 0.18597961962223053, + "step": 2849 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1974.0, + "completions/mean_length": 797.998046875, + "completions/mean_terminated_length": 765.432861328125, + "completions/min_length": 122.0, + "completions/min_terminated_length": 122.0, + "epoch": 0.9729452931637791, + "grad_norm": 2.272385358810425, + "kl": 6.9765625, + "learning_rate": 1.0204227022406866e-07, + "loss": 0.4145, + "num_tokens": 1511546640.0, + "reward": 1.92626953125, + "reward_std": 0.5484147071838379, + "rewards/accuracy_reward/mean": 0.126953125, + "rewards/accuracy_reward/std": 0.33324605226516724, + "rewards/format_reward/mean": 0.8671875, + "rewards/format_reward/std": 0.33970388770103455, + "rewards/tag_count_reward/mean": 0.93212890625, + "rewards/tag_count_reward/std": 0.18874379992485046, + "step": 2850 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 776.94140625, + "completions/mean_terminated_length": 733.2889404296875, + "completions/min_length": 91.0, + "completions/min_terminated_length": 91.0, + "epoch": 0.9732866774771699, + "grad_norm": 1.8062188625335693, + "kl": 6.7109375, + "learning_rate": 1.019915700318121e-07, + "loss": 0.3848, + "num_tokens": 1512033570.0, + "reward": 1.8427734375, + "reward_std": 0.5338236093521118, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.9169921875, + "rewards/tag_count_reward/std": 0.21086981892585754, + "step": 2851 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1883.0, + "completions/mean_length": 815.73046875, + "completions/mean_terminated_length": 760.404052734375, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.9736280617905607, + "grad_norm": 3.596667528152466, + "kl": 9.1953125, + "learning_rate": 1.0194150570558e-07, + "loss": 0.5622, + "num_tokens": 1512537224.0, + "reward": 1.80029296875, + "reward_std": 0.6002273559570312, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.818359375, + "rewards/format_reward/std": 0.38592514395713806, + "rewards/tag_count_reward/mean": 0.90771484375, + "rewards/tag_count_reward/std": 0.21251004934310913, + "step": 2852 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.060546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1945.0, + "completions/mean_length": 808.33203125, + "completions/mean_terminated_length": 728.4365844726562, + "completions/min_length": 157.0, + "completions/min_terminated_length": 157.0, + "epoch": 0.9739694461039515, + "grad_norm": 3.8079118728637695, + "kl": 11.296875, + "learning_rate": 1.0189207731642956e-07, + "loss": 0.6901, + "num_tokens": 1513032450.0, + "reward": 1.8154296875, + "reward_std": 0.6169764995574951, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.814453125, + "rewards/format_reward/std": 0.38912075757980347, + "rewards/tag_count_reward/mean": 0.9052734375, + "rewards/tag_count_reward/std": 0.21965888142585754, + "step": 2853 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1882.0, + "completions/mean_length": 798.49609375, + "completions/mean_terminated_length": 750.3407592773438, + "completions/min_length": 85.0, + "completions/min_terminated_length": 85.0, + "epoch": 0.9743108304173423, + "grad_norm": 1.598483681678772, + "kl": 7.2265625, + "learning_rate": 1.0184328493451527e-07, + "loss": 0.4716, + "num_tokens": 1513523904.0, + "reward": 1.81005859375, + "reward_std": 0.4909866154193878, + "rewards/accuracy_reward/mean": 0.02734375, + "rewards/accuracy_reward/std": 0.16324250400066376, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.92724609375, + "rewards/tag_count_reward/std": 0.19460642337799072, + "step": 2854 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01171875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2027.0, + "completions/mean_length": 740.95703125, + "completions/mean_terminated_length": 725.4585571289062, + "completions/min_length": 117.0, + "completions/min_terminated_length": 117.0, + "epoch": 0.9746522147307332, + "grad_norm": 2.422755002975464, + "kl": 5.1875, + "learning_rate": 1.0179512862908905e-07, + "loss": 0.3305, + "num_tokens": 1513980282.0, + "reward": 1.892578125, + "reward_std": 0.48154059052467346, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.880859375, + "rewards/format_reward/std": 0.32427072525024414, + "rewards/tag_count_reward/mean": 0.943359375, + "rewards/tag_count_reward/std": 0.16541723906993866, + "step": 2855 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2006.0, + "completions/mean_length": 754.169921875, + "completions/mean_terminated_length": 684.95263671875, + "completions/min_length": 58.0, + "completions/min_terminated_length": 58.0, + "epoch": 0.974993599044124, + "grad_norm": 1.3403568267822266, + "kl": 6.9609375, + "learning_rate": 1.0174760846849994e-07, + "loss": 0.4621, + "num_tokens": 1514442305.0, + "reward": 1.89453125, + "reward_std": 0.5234760046005249, + "rewards/accuracy_reward/mean": 0.111328125, + "rewards/accuracy_reward/std": 0.31484565138816833, + "rewards/format_reward/mean": 0.86328125, + "rewards/format_reward/std": 0.3438861668109894, + "rewards/tag_count_reward/mean": 0.919921875, + "rewards/tag_count_reward/std": 0.21026401221752167, + "step": 2856 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1935.0, + "completions/mean_length": 711.4296875, + "completions/mean_terminated_length": 673.8554077148438, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "epoch": 0.9753349833575147, + "grad_norm": 1.6096361875534058, + "kl": 5.5625, + "learning_rate": 1.0170072452019414e-07, + "loss": 0.3562, + "num_tokens": 1514891245.0, + "reward": 1.931640625, + "reward_std": 0.5143195986747742, + "rewards/accuracy_reward/mean": 0.119140625, + "rewards/accuracy_reward/std": 0.32427072525024414, + "rewards/format_reward/mean": 0.875, + "rewards/format_reward/std": 0.3310423493385315, + "rewards/tag_count_reward/mean": 0.9375, + "rewards/tag_count_reward/std": 0.18373169004917145, + "step": 2857 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2015.0, + "completions/mean_length": 763.240234375, + "completions/mean_terminated_length": 702.8118286132812, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "epoch": 0.9756763676709055, + "grad_norm": 1.431836724281311, + "kl": 7.6875, + "learning_rate": 1.0165447685071481e-07, + "loss": 0.4733, + "num_tokens": 1515357976.0, + "reward": 1.77490234375, + "reward_std": 0.5884698033332825, + "rewards/accuracy_reward/mean": 0.056640625, + "rewards/accuracy_reward/std": 0.23138070106506348, + "rewards/format_reward/mean": 0.8125, + "rewards/format_reward/std": 0.39069411158561707, + "rewards/tag_count_reward/mean": 0.90576171875, + "rewards/tag_count_reward/std": 0.21791352331638336, + "step": 2858 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1908.0, + "completions/mean_length": 768.263671875, + "completions/mean_terminated_length": 713.529541015625, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "epoch": 0.9760177519842963, + "grad_norm": 1.0789576768875122, + "kl": 6.25390625, + "learning_rate": 1.0160886552570211e-07, + "loss": 0.4158, + "num_tokens": 1515835359.0, + "reward": 1.84130859375, + "reward_std": 0.5203293561935425, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.859375, + "rewards/format_reward/std": 0.3479743003845215, + "rewards/tag_count_reward/mean": 0.93310546875, + "rewards/tag_count_reward/std": 0.18583056330680847, + "step": 2859 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2038.0, + "completions/mean_length": 752.947265625, + "completions/mean_terminated_length": 734.9960327148438, + "completions/min_length": 135.0, + "completions/min_terminated_length": 135.0, + "epoch": 0.9763591362976871, + "grad_norm": 1.5350780487060547, + "kl": 5.265625, + "learning_rate": 1.0156389060989289e-07, + "loss": 0.316, + "num_tokens": 1516295796.0, + "reward": 1.85546875, + "reward_std": 0.47244903445243835, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.935546875, + "rewards/tag_count_reward/std": 0.17968250811100006, + "step": 2860 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1954.0, + "completions/mean_length": 716.34375, + "completions/mean_terminated_length": 684.384033203125, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "epoch": 0.9767005206110779, + "grad_norm": 2.4552807807922363, + "kl": 5.6875, + "learning_rate": 1.0151955216712089e-07, + "loss": 0.387, + "num_tokens": 1516737892.0, + "reward": 1.90966796875, + "reward_std": 0.5338730812072754, + "rewards/accuracy_reward/mean": 0.099609375, + "rewards/accuracy_reward/std": 0.29977133870124817, + "rewards/format_reward/mean": 0.875, + "rewards/format_reward/std": 0.3310423493385315, + "rewards/tag_count_reward/mean": 0.93505859375, + "rewards/tag_count_reward/std": 0.18187542259693146, + "step": 2861 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1972.0, + "completions/mean_length": 753.84375, + "completions/mean_terminated_length": 725.4291381835938, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "epoch": 0.9770419049244687, + "grad_norm": 1.608132004737854, + "kl": 4.703125, + "learning_rate": 1.0147585026031632e-07, + "loss": 0.3193, + "num_tokens": 1517204996.0, + "reward": 1.900390625, + "reward_std": 0.4727388918399811, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.88671875, + "rewards/format_reward/std": 0.3172462284564972, + "rewards/tag_count_reward/mean": 0.94140625, + "rewards/tag_count_reward/std": 0.16840559244155884, + "step": 2862 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1962.0, + "completions/mean_length": 729.15234375, + "completions/mean_terminated_length": 697.5000610351562, + "completions/min_length": 119.0, + "completions/min_terminated_length": 119.0, + "epoch": 0.9773832892378596, + "grad_norm": 2.0549094676971436, + "kl": 4.421875, + "learning_rate": 1.0143278495150619e-07, + "loss": 0.3088, + "num_tokens": 1517655778.0, + "reward": 1.923828125, + "reward_std": 0.45604830980300903, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.8984375, + "rewards/format_reward/std": 0.30236753821372986, + "rewards/tag_count_reward/mean": 0.94921875, + "rewards/tag_count_reward/std": 0.16361670196056366, + "step": 2863 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1929.0, + "completions/mean_length": 786.349609375, + "completions/mean_terminated_length": 756.0700073242188, + "completions/min_length": 121.0, + "completions/min_terminated_length": 121.0, + "epoch": 0.9777246735512504, + "grad_norm": 0.9248939752578735, + "kl": 5.64453125, + "learning_rate": 1.0139035630181373e-07, + "loss": 0.3801, + "num_tokens": 1518136645.0, + "reward": 1.91796875, + "reward_std": 0.5349627733230591, + "rewards/accuracy_reward/mean": 0.115234375, + "rewards/accuracy_reward/std": 0.3196168541908264, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.93359375, + "rewards/tag_count_reward/std": 0.17965060472488403, + "step": 2864 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 752.0703125, + "completions/mean_terminated_length": 718.3086547851562, + "completions/min_length": 89.0, + "completions/min_terminated_length": 89.0, + "epoch": 0.9780660578646411, + "grad_norm": 1.2031464576721191, + "kl": 4.890625, + "learning_rate": 1.0134856437145871e-07, + "loss": 0.3152, + "num_tokens": 1518591289.0, + "reward": 1.90087890625, + "reward_std": 0.5522708892822266, + "rewards/accuracy_reward/mean": 0.109375, + "rewards/accuracy_reward/std": 0.31241437792778015, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.93017578125, + "rewards/tag_count_reward/std": 0.1918918341398239, + "step": 2865 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1955.0, + "completions/mean_length": 767.494140625, + "completions/mean_terminated_length": 726.1875, + "completions/min_length": 27.0, + "completions/min_terminated_length": 27.0, + "epoch": 0.9784074421780319, + "grad_norm": 1.9690607786178589, + "kl": 4.76953125, + "learning_rate": 1.0130740921975706e-07, + "loss": 0.3667, + "num_tokens": 1519071318.0, + "reward": 1.9267578125, + "reward_std": 0.48103824257850647, + "rewards/accuracy_reward/mean": 0.10685484111309052, + "rewards/accuracy_reward/std": 0.30924052000045776, + "rewards/format_reward/mean": 0.880859375, + "rewards/format_reward/std": 0.32427072525024414, + "rewards/tag_count_reward/mean": 0.9423828125, + "rewards/tag_count_reward/std": 0.17374257743358612, + "step": 2866 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2026.0, + "completions/mean_length": 843.419921875, + "completions/mean_terminated_length": 799.5283813476562, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "epoch": 0.9787488264914227, + "grad_norm": 1.9435549974441528, + "kl": 7.453125, + "learning_rate": 1.0126689090512102e-07, + "loss": 0.4961, + "num_tokens": 1519578557.0, + "reward": 1.857421875, + "reward_std": 0.550590455532074, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.857421875, + "rewards/format_reward/std": 0.3499840497970581, + "rewards/tag_count_reward/mean": 0.92578125, + "rewards/tag_count_reward/std": 0.19933690130710602, + "step": 2867 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1998.0, + "completions/mean_length": 740.43359375, + "completions/mean_terminated_length": 709.0520629882812, + "completions/min_length": 106.0, + "completions/min_terminated_length": 106.0, + "epoch": 0.9790902108048135, + "grad_norm": 1.3769993782043457, + "kl": 5.890625, + "learning_rate": 1.0122700948505894e-07, + "loss": 0.3869, + "num_tokens": 1520039627.0, + "reward": 1.93310546875, + "reward_std": 0.5323408842086792, + "rewards/accuracy_reward/mean": 0.11088709533214569, + "rewards/accuracy_reward/std": 0.3143092691898346, + "rewards/format_reward/mean": 0.884765625, + "rewards/format_reward/std": 0.3196168541908264, + "rewards/tag_count_reward/mean": 0.94091796875, + "rewards/tag_count_reward/std": 0.17570249736309052, + "step": 2868 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1829.0, + "completions/mean_length": 773.1796875, + "completions/mean_terminated_length": 715.9428100585938, + "completions/min_length": 119.0, + "completions/min_terminated_length": 119.0, + "epoch": 0.9794315951182043, + "grad_norm": 2.0342190265655518, + "kl": 8.0078125, + "learning_rate": 1.0118776501617519e-07, + "loss": 0.5134, + "num_tokens": 1520508999.0, + "reward": 1.85107421875, + "reward_std": 0.5643364191055298, + "rewards/accuracy_reward/mean": 0.0703125, + "rewards/accuracy_reward/std": 0.25592297315597534, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.92529296875, + "rewards/tag_count_reward/std": 0.19574713706970215, + "step": 2869 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1845.0, + "completions/mean_length": 710.8828125, + "completions/mean_terminated_length": 673.2931518554688, + "completions/min_length": 135.0, + "completions/min_terminated_length": 135.0, + "epoch": 0.9797729794315951, + "grad_norm": 0.9224192500114441, + "kl": 6.6953125, + "learning_rate": 1.0114915755417014e-07, + "loss": 0.4794, + "num_tokens": 1520960203.0, + "reward": 1.88720703125, + "reward_std": 0.4814002513885498, + "rewards/accuracy_reward/mean": 0.06653226166963577, + "rewards/accuracy_reward/std": 0.2494617998600006, + "rewards/format_reward/mean": 0.880859375, + "rewards/format_reward/std": 0.32427072525024414, + "rewards/tag_count_reward/mean": 0.94189453125, + "rewards/tag_count_reward/std": 0.17180897295475006, + "step": 2870 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1862.0, + "completions/mean_length": 773.857421875, + "completions/mean_terminated_length": 730.0989990234375, + "completions/min_length": 154.0, + "completions/min_terminated_length": 154.0, + "epoch": 0.980114363744986, + "grad_norm": 3.156883478164673, + "kl": 7.5, + "learning_rate": 1.0111118715383995e-07, + "loss": 0.4313, + "num_tokens": 1521435682.0, + "reward": 1.82568359375, + "reward_std": 0.5020943284034729, + "rewards/accuracy_reward/mean": 0.05078125, + "rewards/accuracy_reward/std": 0.21976542472839355, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.92724609375, + "rewards/tag_count_reward/std": 0.19772392511367798, + "step": 2871 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.009765625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2017.0, + "completions/mean_length": 695.564453125, + "completions/mean_terminated_length": 682.226806640625, + "completions/min_length": 87.0, + "completions/min_terminated_length": 87.0, + "epoch": 0.9804557480583768, + "grad_norm": 1.7067991495132446, + "kl": 5.765625, + "learning_rate": 1.0107385386907679e-07, + "loss": 0.3465, + "num_tokens": 1521868387.0, + "reward": 1.9130859375, + "reward_std": 0.4521946907043457, + "rewards/accuracy_reward/mean": 0.080078125, + "rewards/accuracy_reward/std": 0.271679550409317, + "rewards/format_reward/mean": 0.88671875, + "rewards/format_reward/std": 0.3172462284564972, + "rewards/tag_count_reward/mean": 0.9462890625, + "rewards/tag_count_reward/std": 0.16565679013729095, + "step": 2872 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1994.0, + "completions/mean_length": 695.802734375, + "completions/mean_terminated_length": 668.8665771484375, + "completions/min_length": 99.0, + "completions/min_terminated_length": 99.0, + "epoch": 0.9807971323717675, + "grad_norm": 1.465183138847351, + "kl": 7.10546875, + "learning_rate": 1.0103715775286826e-07, + "loss": 0.4461, + "num_tokens": 1522299438.0, + "reward": 1.90576171875, + "reward_std": 0.500202476978302, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.94287109375, + "rewards/tag_count_reward/std": 0.1663554310798645, + "step": 2873 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0546875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1973.0, + "completions/mean_length": 757.095703125, + "completions/mean_terminated_length": 682.415283203125, + "completions/min_length": 64.0, + "completions/min_terminated_length": 64.0, + "epoch": 0.9811385166851583, + "grad_norm": 1.1074867248535156, + "kl": 8.34375, + "learning_rate": 1.010010988572979e-07, + "loss": 0.5809, + "num_tokens": 1522761615.0, + "reward": 1.89892578125, + "reward_std": 0.5496411323547363, + "rewards/accuracy_reward/mean": 0.123046875, + "rewards/accuracy_reward/std": 0.32881227135658264, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.92236328125, + "rewards/tag_count_reward/std": 0.20260746777057648, + "step": 2874 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 783.8515625, + "completions/mean_terminated_length": 761.2325439453125, + "completions/min_length": 125.0, + "completions/min_terminated_length": 125.0, + "epoch": 0.9814799009985491, + "grad_norm": 1.0652945041656494, + "kl": 6.4921875, + "learning_rate": 1.0096567723354473e-07, + "loss": 0.4239, + "num_tokens": 1523248979.0, + "reward": 1.9462890625, + "reward_std": 0.5402163863182068, + "rewards/accuracy_reward/mean": 0.14717741310596466, + "rewards/accuracy_reward/std": 0.3546403646469116, + "rewards/format_reward/mean": 0.8671875, + "rewards/format_reward/std": 0.33970388770103455, + "rewards/tag_count_reward/mean": 0.9365234375, + "rewards/tag_count_reward/std": 0.1779806911945343, + "step": 2875 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1934.0, + "completions/mean_length": 786.330078125, + "completions/mean_terminated_length": 743.0000610351562, + "completions/min_length": 104.0, + "completions/min_terminated_length": 104.0, + "epoch": 0.9818212853119399, + "grad_norm": 1.4906625747680664, + "kl": 6.2265625, + "learning_rate": 1.0093089293188319e-07, + "loss": 0.3868, + "num_tokens": 1523725580.0, + "reward": 1.86181640625, + "reward_std": 0.5163675546646118, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.93017578125, + "rewards/tag_count_reward/std": 0.18606694042682648, + "step": 2876 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2014.0, + "completions/mean_length": 720.775390625, + "completions/mean_terminated_length": 699.7083740234375, + "completions/min_length": 89.0, + "completions/min_terminated_length": 89.0, + "epoch": 0.9821626696253307, + "grad_norm": 2.050997495651245, + "kl": 4.58984375, + "learning_rate": 1.0089674600168329e-07, + "loss": 0.3026, + "num_tokens": 1524170697.0, + "reward": 1.88720703125, + "reward_std": 0.48582902550697327, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.87890625, + "rewards/format_reward/std": 0.3265552520751953, + "rewards/tag_count_reward/mean": 0.93994140625, + "rewards/tag_count_reward/std": 0.18018634617328644, + "step": 2877 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.013671875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2047.0, + "completions/mean_length": 719.587890625, + "completions/mean_terminated_length": 701.1742553710938, + "completions/min_length": 110.0, + "completions/min_terminated_length": 110.0, + "epoch": 0.9825040539387215, + "grad_norm": 1.5940486192703247, + "kl": 3.71875, + "learning_rate": 1.0086323649141032e-07, + "loss": 0.253, + "num_tokens": 1524615382.0, + "reward": 1.9189453125, + "reward_std": 0.4246072769165039, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.90625, + "rewards/format_reward/std": 0.29176566004753113, + "rewards/tag_count_reward/mean": 0.9580078125, + "rewards/tag_count_reward/std": 0.1448541134595871, + "step": 2878 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1929.0, + "completions/mean_length": 691.673828125, + "completions/mean_terminated_length": 647.9213256835938, + "completions/min_length": 76.0, + "completions/min_terminated_length": 76.0, + "epoch": 0.9828454382521123, + "grad_norm": 2.482257127761841, + "kl": 7.109375, + "learning_rate": 1.0083036444862492e-07, + "loss": 0.4532, + "num_tokens": 1525047151.0, + "reward": 1.89208984375, + "reward_std": 0.4856886863708496, + "rewards/accuracy_reward/mean": 0.08984375, + "rewards/accuracy_reward/std": 0.2862374484539032, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.93310546875, + "rewards/tag_count_reward/std": 0.1838454306125641, + "step": 2879 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 754.173828125, + "completions/mean_terminated_length": 715.1246948242188, + "completions/min_length": 113.0, + "completions/min_terminated_length": 113.0, + "epoch": 0.9831868225655032, + "grad_norm": 1.0496032238006592, + "kl": 5.890625, + "learning_rate": 1.0079812991998291e-07, + "loss": 0.406, + "num_tokens": 1525501608.0, + "reward": 1.8701171875, + "reward_std": 0.49785301089286804, + "rewards/accuracy_reward/mean": 0.06854838877916336, + "rewards/accuracy_reward/std": 0.25293970108032227, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.9326171875, + "rewards/tag_count_reward/std": 0.18859504163265228, + "step": 2880 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.021484375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1853.0, + "completions/mean_length": 732.9921875, + "completions/mean_terminated_length": 704.1197509765625, + "completions/min_length": 99.0, + "completions/min_terminated_length": 99.0, + "epoch": 0.9835282068788939, + "grad_norm": 1.1123335361480713, + "kl": 4.94140625, + "learning_rate": 1.0076653295123537e-07, + "loss": 0.2844, + "num_tokens": 1525959044.0, + "reward": 1.95166015625, + "reward_std": 0.4679659307003021, + "rewards/accuracy_reward/mean": 0.130859375, + "rewards/accuracy_reward/std": 0.33757632970809937, + "rewards/format_reward/mean": 0.875, + "rewards/format_reward/std": 0.3310423493385315, + "rewards/tag_count_reward/mean": 0.94580078125, + "rewards/tag_count_reward/std": 0.16438506543636322, + "step": 2881 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.048828125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1973.0, + "completions/mean_length": 806.01953125, + "completions/mean_terminated_length": 742.2628784179688, + "completions/min_length": 53.0, + "completions/min_terminated_length": 53.0, + "epoch": 0.9838695911922847, + "grad_norm": 1.9855237007141113, + "kl": 7.71875, + "learning_rate": 1.0073557358722834e-07, + "loss": 0.4995, + "num_tokens": 1526445614.0, + "reward": 1.85986328125, + "reward_std": 0.5459388494491577, + "rewards/accuracy_reward/mean": 0.06640625, + "rewards/accuracy_reward/std": 0.2492343932390213, + "rewards/format_reward/mean": 0.8671875, + "rewards/format_reward/std": 0.33970388770103455, + "rewards/tag_count_reward/mean": 0.92626953125, + "rewards/tag_count_reward/std": 0.20165254175662994, + "step": 2882 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2037.0, + "completions/mean_length": 750.86328125, + "completions/mean_terminated_length": 700.8721923828125, + "completions/min_length": 4.0, + "completions/min_terminated_length": 4.0, + "epoch": 0.9842109755056755, + "grad_norm": 1.1459872722625732, + "kl": 5.82421875, + "learning_rate": 1.0070525187190301e-07, + "loss": 0.3703, + "num_tokens": 1526912424.0, + "reward": 1.828125, + "reward_std": 0.5574748516082764, + "rewards/accuracy_reward/mean": 0.060483869165182114, + "rewards/accuracy_reward/std": 0.2386218160390854, + "rewards/format_reward/mean": 0.84375, + "rewards/format_reward/std": 0.36344730854034424, + "rewards/tag_count_reward/mean": 0.92578125, + "rewards/tag_count_reward/std": 0.1840227097272873, + "step": 2883 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1881.0, + "completions/mean_length": 776.88671875, + "completions/mean_terminated_length": 733.2323608398438, + "completions/min_length": 99.0, + "completions/min_terminated_length": 99.0, + "epoch": 0.9845523598190663, + "grad_norm": 1.5667763948440552, + "kl": 6.0234375, + "learning_rate": 1.0067556784829557e-07, + "loss": 0.4198, + "num_tokens": 1527396206.0, + "reward": 1.87841796875, + "reward_std": 0.5296695828437805, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.93115234375, + "rewards/tag_count_reward/std": 0.18643119931221008, + "step": 2884 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1994.0, + "completions/mean_length": 764.12890625, + "completions/mean_terminated_length": 714.6490478515625, + "completions/min_length": 92.0, + "completions/min_terminated_length": 92.0, + "epoch": 0.9848937441324571, + "grad_norm": 1.4942023754119873, + "kl": 7.328125, + "learning_rate": 1.0064652155853695e-07, + "loss": 0.5038, + "num_tokens": 1527870544.0, + "reward": 1.8798828125, + "reward_std": 0.5620890855789185, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.85546875, + "rewards/format_reward/std": 0.35197147727012634, + "rewards/tag_count_reward/mean": 0.9287109375, + "rewards/tag_count_reward/std": 0.1929427534341812, + "step": 2885 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1991.0, + "completions/mean_length": 756.03125, + "completions/mean_terminated_length": 703.5121459960938, + "completions/min_length": 95.0, + "completions/min_terminated_length": 95.0, + "epoch": 0.9852351284458479, + "grad_norm": 1.0311920642852783, + "kl": 6.7734375, + "learning_rate": 1.0061811304385314e-07, + "loss": 0.4419, + "num_tokens": 1528338144.0, + "reward": 1.83447265625, + "reward_std": 0.494672030210495, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.849609375, + "rewards/format_reward/std": 0.35780346393585205, + "rewards/tag_count_reward/mean": 0.93017578125, + "rewards/tag_count_reward/std": 0.1820801943540573, + "step": 2886 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1803.0, + "completions/mean_length": 758.474609375, + "completions/mean_terminated_length": 706.0548706054688, + "completions/min_length": 113.0, + "completions/min_terminated_length": 113.0, + "epoch": 0.9855765127592387, + "grad_norm": 1.2463688850402832, + "kl": 6.3359375, + "learning_rate": 1.0059034234456476e-07, + "loss": 0.3977, + "num_tokens": 1528801571.0, + "reward": 1.857421875, + "reward_std": 0.5183749198913574, + "rewards/accuracy_reward/mean": 0.0546875, + "rewards/accuracy_reward/std": 0.2275916188955307, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.93359375, + "rewards/tag_count_reward/std": 0.19655638933181763, + "step": 2887 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1979.0, + "completions/mean_length": 697.787109375, + "completions/mean_terminated_length": 665.3820190429688, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "epoch": 0.9859178970726296, + "grad_norm": 1.701975703239441, + "kl": 5.19140625, + "learning_rate": 1.0056320950008728e-07, + "loss": 0.3726, + "num_tokens": 1529236070.0, + "reward": 1.99853515625, + "reward_std": 0.4950244128704071, + "rewards/accuracy_reward/mean": 0.146484375, + "rewards/accuracy_reward/std": 0.35393697023391724, + "rewards/format_reward/mean": 0.90234375, + "rewards/format_reward/std": 0.29713961482048035, + "rewards/tag_count_reward/mean": 0.94970703125, + "rewards/tag_count_reward/std": 0.16339389979839325, + "step": 2888 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1956.0, + "completions/mean_length": 703.5390625, + "completions/mean_terminated_length": 679.4830932617188, + "completions/min_length": 105.0, + "completions/min_terminated_length": 105.0, + "epoch": 0.9862592813860203, + "grad_norm": 1.230083703994751, + "kl": 5.7109375, + "learning_rate": 1.0053671454893084e-07, + "loss": 0.3703, + "num_tokens": 1529678954.0, + "reward": 1.86181640625, + "reward_std": 0.4793586730957031, + "rewards/accuracy_reward/mean": 0.044921875, + "rewards/accuracy_reward/std": 0.20733514428138733, + "rewards/format_reward/mean": 0.875, + "rewards/format_reward/std": 0.3310423493385315, + "rewards/tag_count_reward/mean": 0.94189453125, + "rewards/tag_count_reward/std": 0.17533229291439056, + "step": 2889 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2001.0, + "completions/mean_length": 769.111328125, + "completions/mean_terminated_length": 730.5130615234375, + "completions/min_length": 83.0, + "completions/min_terminated_length": 83.0, + "epoch": 0.9866006656994111, + "grad_norm": 1.0111032724380493, + "kl": 5.9921875, + "learning_rate": 1.0051085752870009e-07, + "loss": 0.3956, + "num_tokens": 1530148531.0, + "reward": 1.89794921875, + "reward_std": 0.4716569185256958, + "rewards/accuracy_reward/mean": 0.07421875, + "rewards/accuracy_reward/std": 0.2623828947544098, + "rewards/format_reward/mean": 0.880859375, + "rewards/format_reward/std": 0.32427072525024414, + "rewards/tag_count_reward/mean": 0.94287109375, + "rewards/tag_count_reward/std": 0.1749558448791504, + "step": 2890 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1983.0, + "completions/mean_length": 753.181640625, + "completions/mean_terminated_length": 719.4489135742188, + "completions/min_length": 111.0, + "completions/min_terminated_length": 111.0, + "epoch": 0.9869420500128019, + "grad_norm": 1.263071894645691, + "kl": 6.453125, + "learning_rate": 1.0048563847609443e-07, + "loss": 0.4032, + "num_tokens": 1530612112.0, + "reward": 1.93896484375, + "reward_std": 0.5125135183334351, + "rewards/accuracy_reward/mean": 0.115234375, + "rewards/accuracy_reward/std": 0.3196168541908264, + "rewards/format_reward/mean": 0.884765625, + "rewards/format_reward/std": 0.3196168541908264, + "rewards/tag_count_reward/mean": 0.93896484375, + "rewards/tag_count_reward/std": 0.1784919947385788, + "step": 2891 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2007.0, + "completions/mean_length": 825.1875, + "completions/mean_terminated_length": 778.0608520507812, + "completions/min_length": 107.0, + "completions/min_terminated_length": 107.0, + "epoch": 0.9872834343261927, + "grad_norm": 1.4755581617355347, + "kl": 6.1953125, + "learning_rate": 1.0046105742690761e-07, + "loss": 0.3688, + "num_tokens": 1531113040.0, + "reward": 1.87890625, + "reward_std": 0.4649357497692108, + "rewards/accuracy_reward/mean": 0.058467742055654526, + "rewards/accuracy_reward/std": 0.23486268520355225, + "rewards/format_reward/mean": 0.8828125, + "rewards/format_reward/std": 0.32195815443992615, + "rewards/tag_count_reward/mean": 0.939453125, + "rewards/tag_count_reward/std": 0.17900052666664124, + "step": 2892 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1985.0, + "completions/mean_length": 762.21484375, + "completions/mean_terminated_length": 712.6612548828125, + "completions/min_length": 86.0, + "completions/min_terminated_length": 86.0, + "epoch": 0.9876248186395835, + "grad_norm": 1.8035228252410889, + "kl": 8.5078125, + "learning_rate": 1.0043711441602797e-07, + "loss": 0.5307, + "num_tokens": 1531586462.0, + "reward": 1.85302734375, + "reward_std": 0.5432873964309692, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.861328125, + "rewards/format_reward/std": 0.34594178199768066, + "rewards/tag_count_reward/mean": 0.93310546875, + "rewards/tag_count_reward/std": 0.1864875704050064, + "step": 2893 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0234375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2013.0, + "completions/mean_length": 752.18359375, + "completions/mean_terminated_length": 721.0840454101562, + "completions/min_length": 147.0, + "completions/min_terminated_length": 147.0, + "epoch": 0.9879662029529743, + "grad_norm": 1.287606954574585, + "kl": 6.2734375, + "learning_rate": 1.0041380947743828e-07, + "loss": 0.3622, + "num_tokens": 1532047356.0, + "reward": 1.9140625, + "reward_std": 0.5265927314758301, + "rewards/accuracy_reward/mean": 0.103515625, + "rewards/accuracy_reward/std": 0.30492907762527466, + "rewards/format_reward/mean": 0.87109375, + "rewards/format_reward/std": 0.33542385697364807, + "rewards/tag_count_reward/mean": 0.939453125, + "rewards/tag_count_reward/std": 0.17900052666664124, + "step": 2894 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1957.0, + "completions/mean_length": 745.78515625, + "completions/mean_terminated_length": 703.7781982421875, + "completions/min_length": 108.0, + "completions/min_terminated_length": 108.0, + "epoch": 0.9883075872663651, + "grad_norm": 1.184757947921753, + "kl": 6.5625, + "learning_rate": 1.0039114264421555e-07, + "loss": 0.4269, + "num_tokens": 1532508270.0, + "reward": 1.94384765625, + "reward_std": 0.5128960609436035, + "rewards/accuracy_reward/mean": 0.115234375, + "rewards/accuracy_reward/std": 0.3196168541908264, + "rewards/format_reward/mean": 0.884765625, + "rewards/format_reward/std": 0.3196168541908264, + "rewards/tag_count_reward/mean": 0.94384765625, + "rewards/tag_count_reward/std": 0.16814936697483063, + "step": 2895 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.052734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2029.0, + "completions/mean_length": 739.978515625, + "completions/mean_terminated_length": 667.1608276367188, + "completions/min_length": 68.0, + "completions/min_terminated_length": 68.0, + "epoch": 0.988648971579756, + "grad_norm": 3.383671522140503, + "kl": 11.640625, + "learning_rate": 1.0036911394853133e-07, + "loss": 0.7671, + "num_tokens": 1532957027.0, + "reward": 1.84912109375, + "reward_std": 0.5863605737686157, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.84765625, + "rewards/format_reward/std": 0.35970520973205566, + "rewards/tag_count_reward/mean": 0.91943359375, + "rewards/tag_count_reward/std": 0.20978572964668274, + "step": 2896 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1938.0, + "completions/mean_length": 730.890625, + "completions/mean_terminated_length": 707.3240356445312, + "completions/min_length": 118.0, + "completions/min_terminated_length": 118.0, + "epoch": 0.9889903558931467, + "grad_norm": 1.3612533807754517, + "kl": 5.4765625, + "learning_rate": 1.0034772342165116e-07, + "loss": 0.3259, + "num_tokens": 1533404635.0, + "reward": 1.93408203125, + "reward_std": 0.4663509726524353, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.890625, + "rewards/format_reward/std": 0.31241437792778015, + "rewards/tag_count_reward/mean": 0.94775390625, + "rewards/tag_count_reward/std": 0.16427458822727203, + "step": 2897 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2025.0, + "completions/mean_length": 807.328125, + "completions/mean_terminated_length": 748.973388671875, + "completions/min_length": 110.0, + "completions/min_terminated_length": 110.0, + "epoch": 0.9893317402065375, + "grad_norm": 1.000133752822876, + "kl": 7.7578125, + "learning_rate": 1.003269710939351e-07, + "loss": 0.5293, + "num_tokens": 1533906179.0, + "reward": 1.82861328125, + "reward_std": 0.46436357498168945, + "rewards/accuracy_reward/mean": 0.015625, + "rewards/accuracy_reward/std": 0.12414088100194931, + "rewards/format_reward/mean": 0.876953125, + "rewards/format_reward/std": 0.32881227135658264, + "rewards/tag_count_reward/mean": 0.93603515625, + "rewards/tag_count_reward/std": 0.18751464784145355, + "step": 2898 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1897.0, + "completions/mean_length": 796.3671875, + "completions/mean_terminated_length": 755.991943359375, + "completions/min_length": 136.0, + "completions/min_terminated_length": 136.0, + "epoch": 0.9896731245199283, + "grad_norm": 1.1438095569610596, + "kl": 7.9609375, + "learning_rate": 1.0030685699483725e-07, + "loss": 0.5112, + "num_tokens": 1534384527.0, + "reward": 1.826171875, + "reward_std": 0.5321215391159058, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.849609375, + "rewards/format_reward/std": 0.35780346393585205, + "rewards/tag_count_reward/mean": 0.927734375, + "rewards/tag_count_reward/std": 0.1880800724029541, + "step": 2899 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2048.0, + "completions/mean_length": 812.654296875, + "completions/mean_terminated_length": 757.1897583007812, + "completions/min_length": 124.0, + "completions/min_terminated_length": 124.0, + "epoch": 0.9900145088333191, + "grad_norm": 1.2878631353378296, + "kl": 7.3203125, + "learning_rate": 1.0028738115290592e-07, + "loss": 0.4888, + "num_tokens": 1534874670.0, + "reward": 1.83740234375, + "reward_std": 0.5386607050895691, + "rewards/accuracy_reward/mean": 0.048828125, + "rewards/accuracy_reward/std": 0.2157193273305893, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.92333984375, + "rewards/tag_count_reward/std": 0.20774433016777039, + "step": 2900 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2003.0, + "completions/mean_length": 842.08984375, + "completions/mean_terminated_length": 800.6748046875, + "completions/min_length": 73.0, + "completions/min_terminated_length": 73.0, + "epoch": 0.9903558931467099, + "grad_norm": 1.9414771795272827, + "kl": 5.9765625, + "learning_rate": 1.0026854359578345e-07, + "loss": 0.4032, + "num_tokens": 1535386572.0, + "reward": 1.8466796875, + "reward_std": 0.47419971227645874, + "rewards/accuracy_reward/mean": 0.033203125, + "rewards/accuracy_reward/std": 0.17934183776378632, + "rewards/format_reward/mean": 0.876953125, + "rewards/format_reward/std": 0.32881227135658264, + "rewards/tag_count_reward/mean": 0.9365234375, + "rewards/tag_count_reward/std": 0.19250644743442535, + "step": 2901 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.041015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2023.0, + "completions/mean_length": 779.513671875, + "completions/mean_terminated_length": 725.2607421875, + "completions/min_length": 123.0, + "completions/min_terminated_length": 123.0, + "epoch": 0.9906972774601007, + "grad_norm": 1.2120583057403564, + "kl": 7.15625, + "learning_rate": 1.0025034435020633e-07, + "loss": 0.4822, + "num_tokens": 1535863731.0, + "reward": 1.83740234375, + "reward_std": 0.5296105742454529, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.853515625, + "rewards/format_reward/std": 0.35393697023391724, + "rewards/tag_count_reward/mean": 0.92333984375, + "rewards/tag_count_reward/std": 0.1999441683292389, + "step": 2902 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.04296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2035.0, + "completions/mean_length": 838.51953125, + "completions/mean_terminated_length": 784.21630859375, + "completions/min_length": 45.0, + "completions/min_terminated_length": 45.0, + "epoch": 0.9910386617734915, + "grad_norm": 1.3991584777832031, + "kl": 7.125, + "learning_rate": 1.0023278344200509e-07, + "loss": 0.4354, + "num_tokens": 1536377341.0, + "reward": 1.7880859375, + "reward_std": 0.5258431434631348, + "rewards/accuracy_reward/mean": 0.04296875, + "rewards/accuracy_reward/std": 0.2029850035905838, + "rewards/format_reward/mean": 0.8359375, + "rewards/format_reward/std": 0.37069445848464966, + "rewards/tag_count_reward/mean": 0.9091796875, + "rewards/tag_count_reward/std": 0.2245972901582718, + "step": 2903 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1929.0, + "completions/mean_length": 720.208984375, + "completions/mean_terminated_length": 696.4512329101562, + "completions/min_length": 131.0, + "completions/min_terminated_length": 131.0, + "epoch": 0.9913800460868823, + "grad_norm": 2.394346237182617, + "kl": 5.46484375, + "learning_rate": 1.0021586089610422e-07, + "loss": 0.3727, + "num_tokens": 1536825368.0, + "reward": 1.91650390625, + "reward_std": 0.47387444972991943, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.904296875, + "rewards/format_reward/std": 0.2944713830947876, + "rewards/tag_count_reward/mean": 0.94775390625, + "rewards/tag_count_reward/std": 0.17084409296512604, + "step": 2904 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.017578125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1990.0, + "completions/mean_length": 725.705078125, + "completions/mean_terminated_length": 702.0457153320312, + "completions/min_length": 133.0, + "completions/min_terminated_length": 133.0, + "epoch": 0.9917214304002732, + "grad_norm": 2.2166171073913574, + "kl": 4.75, + "learning_rate": 1.0019957673652214e-07, + "loss": 0.3301, + "num_tokens": 1537265905.0, + "reward": 1.876953125, + "reward_std": 0.44129854440689087, + "rewards/accuracy_reward/mean": 0.02734375, + "rewards/accuracy_reward/std": 0.16324250400066376, + "rewards/format_reward/mean": 0.90234375, + "rewards/format_reward/std": 0.29713961482048035, + "rewards/tag_count_reward/mean": 0.947265625, + "rewards/tag_count_reward/std": 0.17318345606327057, + "step": 2905 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1981.0, + "completions/mean_length": 736.302734375, + "completions/mean_terminated_length": 699.4276733398438, + "completions/min_length": 109.0, + "completions/min_terminated_length": 109.0, + "epoch": 0.9920628147136639, + "grad_norm": 2.6191136837005615, + "kl": 5.703125, + "learning_rate": 1.0018393098637125e-07, + "loss": 0.3778, + "num_tokens": 1537722604.0, + "reward": 1.92822265625, + "reward_std": 0.4573560655117035, + "rewards/accuracy_reward/mean": 0.076171875, + "rewards/accuracy_reward/std": 0.26553234457969666, + "rewards/format_reward/mean": 0.904296875, + "rewards/format_reward/std": 0.2944713830947876, + "rewards/tag_count_reward/mean": 0.94775390625, + "rewards/tag_count_reward/std": 0.1729784905910492, + "step": 2906 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.015625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1907.0, + "completions/mean_length": 709.029296875, + "completions/mean_terminated_length": 687.7758178710938, + "completions/min_length": 132.0, + "completions/min_terminated_length": 132.0, + "epoch": 0.9924041990270547, + "grad_norm": 1.049399495124817, + "kl": 4.73046875, + "learning_rate": 1.0016892366785786e-07, + "loss": 0.3121, + "num_tokens": 1538163867.0, + "reward": 1.998046875, + "reward_std": 0.4111006259918213, + "rewards/accuracy_reward/mean": 0.119140625, + "rewards/accuracy_reward/std": 0.32427072525024414, + "rewards/format_reward/mean": 0.919921875, + "rewards/format_reward/std": 0.271679550409317, + "rewards/tag_count_reward/mean": 0.958984375, + "rewards/tag_count_reward/std": 0.14680999517440796, + "step": 2907 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2001.0, + "completions/mean_length": 790.71484375, + "completions/mean_terminated_length": 752.7685546875, + "completions/min_length": 118.0, + "completions/min_terminated_length": 118.0, + "epoch": 0.9927455833404455, + "grad_norm": 1.9230318069458008, + "kl": 5.796875, + "learning_rate": 1.0015455480228208e-07, + "loss": 0.3848, + "num_tokens": 1538642873.0, + "reward": 1.9091796875, + "reward_std": 0.4895654320716858, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.890625, + "rewards/format_reward/std": 0.31241437792778015, + "rewards/tag_count_reward/mean": 0.9365234375, + "rewards/tag_count_reward/std": 0.18930304050445557, + "step": 2908 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1964.0, + "completions/mean_length": 747.640625, + "completions/mean_terminated_length": 721.737060546875, + "completions/min_length": 63.0, + "completions/min_terminated_length": 63.0, + "epoch": 0.9930869676538363, + "grad_norm": 2.3826773166656494, + "kl": 5.78125, + "learning_rate": 1.0014082441003791e-07, + "loss": 0.3571, + "num_tokens": 1539106177.0, + "reward": 1.92529296875, + "reward_std": 0.4951738715171814, + "rewards/accuracy_reward/mean": 0.099609375, + "rewards/accuracy_reward/std": 0.29977133870124817, + "rewards/format_reward/mean": 0.8828125, + "rewards/format_reward/std": 0.32195815443992615, + "rewards/tag_count_reward/mean": 0.94287109375, + "rewards/tag_count_reward/std": 0.17213678359985352, + "step": 2909 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.029296875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2001.0, + "completions/mean_length": 795.3046875, + "completions/mean_terminated_length": 757.4969482421875, + "completions/min_length": 119.0, + "completions/min_terminated_length": 119.0, + "epoch": 0.9934283519672271, + "grad_norm": 2.0867769718170166, + "kl": 7.4140625, + "learning_rate": 1.001277325106131e-07, + "loss": 0.4757, + "num_tokens": 1539599325.0, + "reward": 1.89453125, + "reward_std": 0.5042934417724609, + "rewards/accuracy_reward/mean": 0.064453125, + "rewards/accuracy_reward/std": 0.24579854309558868, + "rewards/format_reward/mean": 0.88671875, + "rewards/format_reward/std": 0.3172462284564972, + "rewards/tag_count_reward/mean": 0.943359375, + "rewards/tag_count_reward/std": 0.1719430834054947, + "step": 2910 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1965.0, + "completions/mean_length": 749.224609375, + "completions/mean_terminated_length": 723.3526000976562, + "completions/min_length": 122.0, + "completions/min_terminated_length": 122.0, + "epoch": 0.9937697362806179, + "grad_norm": 1.966011881828308, + "kl": 7.46875, + "learning_rate": 1.0011527912258924e-07, + "loss": 0.4478, + "num_tokens": 1540065920.0, + "reward": 1.88134765625, + "reward_std": 0.5172063708305359, + "rewards/accuracy_reward/mean": 0.08870967477560043, + "rewards/accuracy_reward/std": 0.2846112847328186, + "rewards/format_reward/mean": 0.859375, + "rewards/format_reward/std": 0.3479743003845215, + "rewards/tag_count_reward/mean": 0.93603515625, + "rewards/tag_count_reward/std": 0.171859011054039, + "step": 2911 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.01953125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1955.0, + "completions/mean_length": 726.9921875, + "completions/mean_terminated_length": 700.6773071289062, + "completions/min_length": 137.0, + "completions/min_terminated_length": 137.0, + "epoch": 0.9941111205940087, + "grad_norm": 2.497357130050659, + "kl": 7.30859375, + "learning_rate": 1.0010346426364161e-07, + "loss": 0.4731, + "num_tokens": 1540513356.0, + "reward": 1.927734375, + "reward_std": 0.49232202768325806, + "rewards/accuracy_reward/mean": 0.09765625, + "rewards/accuracy_reward/std": 0.29713961482048035, + "rewards/format_reward/mean": 0.88671875, + "rewards/format_reward/std": 0.3172462284564972, + "rewards/tag_count_reward/mean": 0.943359375, + "rewards/tag_count_reward/std": 0.1697956770658493, + "step": 2912 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2004.0, + "completions/mean_length": 795.17578125, + "completions/mean_terminated_length": 749.5263061523438, + "completions/min_length": 79.0, + "completions/min_terminated_length": 79.0, + "epoch": 0.9944525049073996, + "grad_norm": 4.3932576179504395, + "kl": 8.859375, + "learning_rate": 1.0009228795053926e-07, + "loss": 0.5182, + "num_tokens": 1540991062.0, + "reward": 1.84814453125, + "reward_std": 0.4987267851829529, + "rewards/accuracy_reward/mean": 0.06854838877916336, + "rewards/accuracy_reward/std": 0.25293970108032227, + "rewards/format_reward/mean": 0.857421875, + "rewards/format_reward/std": 0.3499840497970581, + "rewards/tag_count_reward/mean": 0.92431640625, + "rewards/tag_count_reward/std": 0.1966189742088318, + "step": 2913 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.064453125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2004.0, + "completions/mean_length": 876.916015625, + "completions/mean_terminated_length": 796.2359008789062, + "completions/min_length": 23.0, + "completions/min_terminated_length": 23.0, + "epoch": 0.9947938892207903, + "grad_norm": 5.697507858276367, + "kl": 11.0703125, + "learning_rate": 1.0008175019914494e-07, + "loss": 0.6048, + "num_tokens": 1541517451.0, + "reward": 1.7763671875, + "reward_std": 0.6395107507705688, + "rewards/accuracy_reward/mean": 0.08203125, + "rewards/accuracy_reward/std": 0.2746807038784027, + "rewards/format_reward/mean": 0.802734375, + "rewards/format_reward/std": 0.3983237147331238, + "rewards/tag_count_reward/mean": 0.8916015625, + "rewards/tag_count_reward/std": 0.2345370054244995, + "step": 2914 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.02734375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1987.0, + "completions/mean_length": 797.865234375, + "completions/mean_terminated_length": 762.7208251953125, + "completions/min_length": 83.0, + "completions/min_terminated_length": 83.0, + "epoch": 0.9951352735341811, + "grad_norm": 1.3660556077957153, + "kl": 7.203125, + "learning_rate": 1.0007185102441505e-07, + "loss": 0.4109, + "num_tokens": 1541997958.0, + "reward": 1.8818359375, + "reward_std": 0.5130884647369385, + "rewards/accuracy_reward/mean": 0.068359375, + "rewards/accuracy_reward/std": 0.25260838866233826, + "rewards/format_reward/mean": 0.875, + "rewards/format_reward/std": 0.3310423493385315, + "rewards/tag_count_reward/mean": 0.9384765625, + "rewards/tag_count_reward/std": 0.1779806911945343, + "step": 2915 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2032.0, + "completions/mean_length": 896.453125, + "completions/mean_terminated_length": 819.683349609375, + "completions/min_length": 97.0, + "completions/min_terminated_length": 97.0, + "epoch": 0.9954766578475719, + "grad_norm": 1.6234740018844604, + "kl": 7.8984375, + "learning_rate": 1.0006259044039964e-07, + "loss": 0.5138, + "num_tokens": 1542542478.0, + "reward": 1.88037109375, + "reward_std": 0.5088456869125366, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.87890625, + "rewards/format_reward/std": 0.3265552520751953, + "rewards/tag_count_reward/mean": 0.92919921875, + "rewards/tag_count_reward/std": 0.20027364790439606, + "step": 2916 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.044921875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1997.0, + "completions/mean_length": 780.3984375, + "completions/mean_terminated_length": 720.777099609375, + "completions/min_length": 96.0, + "completions/min_terminated_length": 96.0, + "epoch": 0.9958180421609627, + "grad_norm": 1.4404579401016235, + "kl": 8.1015625, + "learning_rate": 1.000539684602424e-07, + "loss": 0.5381, + "num_tokens": 1543018458.0, + "reward": 1.8310546875, + "reward_std": 0.5409401655197144, + "rewards/accuracy_reward/mean": 0.052734375, + "rewards/accuracy_reward/std": 0.22372129559516907, + "rewards/format_reward/mean": 0.86328125, + "rewards/format_reward/std": 0.3438861668109894, + "rewards/tag_count_reward/mean": 0.9150390625, + "rewards/tag_count_reward/std": 0.2163981944322586, + "step": 2917 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 798.650390625, + "completions/mean_terminated_length": 747.86376953125, + "completions/min_length": 81.0, + "completions/min_terminated_length": 81.0, + "epoch": 0.9961594264743535, + "grad_norm": 1.7061262130737305, + "kl": 6.640625, + "learning_rate": 1.0004598509618068e-07, + "loss": 0.4761, + "num_tokens": 1543498935.0, + "reward": 1.8583984375, + "reward_std": 0.47106894850730896, + "rewards/accuracy_reward/mean": 0.033203125, + "rewards/accuracy_reward/std": 0.17934183776378632, + "rewards/format_reward/mean": 0.884765625, + "rewards/format_reward/std": 0.3196168541908264, + "rewards/tag_count_reward/mean": 0.9404296875, + "rewards/tag_count_reward/std": 0.18337510526180267, + "step": 2918 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.046875, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2019.0, + "completions/mean_length": 861.73046875, + "completions/mean_terminated_length": 803.3892822265625, + "completions/min_length": 190.0, + "completions/min_terminated_length": 190.0, + "epoch": 0.9965008107877443, + "grad_norm": 2.162806510925293, + "kl": 7.6875, + "learning_rate": 1.0003864035954539e-07, + "loss": 0.455, + "num_tokens": 1544016797.0, + "reward": 1.91650390625, + "reward_std": 0.570501446723938, + "rewards/accuracy_reward/mean": 0.13671875, + "rewards/accuracy_reward/std": 0.3438861668109894, + "rewards/format_reward/mean": 0.859375, + "rewards/format_reward/std": 0.3479743003845215, + "rewards/tag_count_reward/mean": 0.92041015625, + "rewards/tag_count_reward/std": 0.2078179121017456, + "step": 2919 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.037109375, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2009.0, + "completions/mean_length": 819.791015625, + "completions/mean_terminated_length": 772.4563598632812, + "completions/min_length": 112.0, + "completions/min_terminated_length": 112.0, + "epoch": 0.9968421951011351, + "grad_norm": 1.647328495979309, + "kl": 5.90625, + "learning_rate": 1.0003193426076107e-07, + "loss": 0.3672, + "num_tokens": 1544526674.0, + "reward": 1.89013671875, + "reward_std": 0.524093747138977, + "rewards/accuracy_reward/mean": 0.095703125, + "rewards/accuracy_reward/std": 0.2944713830947876, + "rewards/format_reward/mean": 0.865234375, + "rewards/format_reward/std": 0.3418070077896118, + "rewards/tag_count_reward/mean": 0.92919921875, + "rewards/tag_count_reward/std": 0.19280590116977692, + "step": 2920 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1975.0, + "completions/mean_length": 832.6796875, + "completions/mean_terminated_length": 751.6583862304688, + "completions/min_length": 107.0, + "completions/min_terminated_length": 107.0, + "epoch": 0.997183579414526, + "grad_norm": 1.7485300302505493, + "kl": 8.65625, + "learning_rate": 1.0002586680934577e-07, + "loss": 0.5546, + "num_tokens": 1545028510.0, + "reward": 1.80126953125, + "reward_std": 0.6211615800857544, + "rewards/accuracy_reward/mean": 0.09375, + "rewards/accuracy_reward/std": 0.29176566004753113, + "rewards/format_reward/mean": 0.8046875, + "rewards/format_reward/std": 0.3968288004398346, + "rewards/tag_count_reward/mean": 0.90283203125, + "rewards/tag_count_reward/std": 0.22384031116962433, + "step": 2921 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.033203125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2040.0, + "completions/mean_length": 821.447265625, + "completions/mean_terminated_length": 779.3232421875, + "completions/min_length": 122.0, + "completions/min_terminated_length": 122.0, + "epoch": 0.9975249637279167, + "grad_norm": 1.8131388425827026, + "kl": 4.9453125, + "learning_rate": 1.0002043801391112e-07, + "loss": 0.3404, + "num_tokens": 1545526915.0, + "reward": 1.8935546875, + "reward_std": 0.46334606409072876, + "rewards/accuracy_reward/mean": 0.072265625, + "rewards/accuracy_reward/std": 0.2591804563999176, + "rewards/format_reward/mean": 0.884765625, + "rewards/format_reward/std": 0.3196168541908264, + "rewards/tag_count_reward/mean": 0.9365234375, + "rewards/tag_count_reward/std": 0.18800638616085052, + "step": 2922 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.025390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1934.0, + "completions/mean_length": 727.15234375, + "completions/mean_terminated_length": 692.7415161132812, + "completions/min_length": 71.0, + "completions/min_terminated_length": 71.0, + "epoch": 0.9978663480413075, + "grad_norm": 1.924739956855774, + "kl": 4.8359375, + "learning_rate": 1.0001564788216237e-07, + "loss": 0.3474, + "num_tokens": 1545969441.0, + "reward": 1.9033203125, + "reward_std": 0.42516008019447327, + "rewards/accuracy_reward/mean": 0.060546875, + "rewards/accuracy_reward/std": 0.2387305200099945, + "rewards/format_reward/mean": 0.89453125, + "rewards/format_reward/std": 0.3074568510055542, + "rewards/tag_count_reward/mean": 0.9482421875, + "rewards/tag_count_reward/std": 0.1670125275850296, + "step": 2923 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.03515625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1925.0, + "completions/mean_length": 788.767578125, + "completions/mean_terminated_length": 742.8846435546875, + "completions/min_length": 140.0, + "completions/min_terminated_length": 140.0, + "epoch": 0.9982077323546983, + "grad_norm": 1.661131501197815, + "kl": 4.83203125, + "learning_rate": 1.0001149642089817e-07, + "loss": 0.3272, + "num_tokens": 1546444058.0, + "reward": 1.916015625, + "reward_std": 0.48726630210876465, + "rewards/accuracy_reward/mean": 0.0859375, + "rewards/accuracy_reward/std": 0.28054583072662354, + "rewards/format_reward/mean": 0.888671875, + "rewards/format_reward/std": 0.31484565138816833, + "rewards/tag_count_reward/mean": 0.94140625, + "rewards/tag_count_reward/std": 0.17270830273628235, + "step": 2924 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.056640625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1980.0, + "completions/mean_length": 869.40234375, + "completions/mean_terminated_length": 798.6376953125, + "completions/min_length": 76.0, + "completions/min_terminated_length": 76.0, + "epoch": 0.9985491166680891, + "grad_norm": 2.2363667488098145, + "kl": 5.9765625, + "learning_rate": 1.0000798363601074e-07, + "loss": 0.431, + "num_tokens": 1546969224.0, + "reward": 1.90380859375, + "reward_std": 0.5548034310340881, + "rewards/accuracy_reward/mean": 0.103515625, + "rewards/accuracy_reward/std": 0.30492907762527466, + "rewards/format_reward/mean": 0.869140625, + "rewards/format_reward/std": 0.33757632970809937, + "rewards/tag_count_reward/mean": 0.93115234375, + "rewards/tag_count_reward/std": 0.19351330399513245, + "step": 2925 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2015.0, + "completions/mean_length": 755.484375, + "completions/mean_terminated_length": 702.9430541992188, + "completions/min_length": 75.0, + "completions/min_terminated_length": 75.0, + "epoch": 0.9988905009814799, + "grad_norm": 3.301025867462158, + "kl": 5.73828125, + "learning_rate": 1.0000510953248593e-07, + "loss": 0.4159, + "num_tokens": 1547430640.0, + "reward": 1.8583984375, + "reward_std": 0.5075932741165161, + "rewards/accuracy_reward/mean": 0.05859375, + "rewards/accuracy_reward/std": 0.23509246110916138, + "rewards/format_reward/mean": 0.8671875, + "rewards/format_reward/std": 0.33970388770103455, + "rewards/tag_count_reward/mean": 0.9326171875, + "rewards/tag_count_reward/std": 0.1879453957080841, + "step": 2926 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.0390625, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1990.0, + "completions/mean_length": 871.13671875, + "completions/mean_terminated_length": 823.2966918945312, + "completions/min_length": 80.0, + "completions/min_terminated_length": 80.0, + "epoch": 0.9992318852948707, + "grad_norm": 2.2923011779785156, + "kl": 6.11328125, + "learning_rate": 1.0000287411440292e-07, + "loss": 0.4166, + "num_tokens": 1547950582.0, + "reward": 1.826171875, + "reward_std": 0.47404032945632935, + "rewards/accuracy_reward/mean": 0.0234375, + "rewards/accuracy_reward/std": 0.15143637359142303, + "rewards/format_reward/mean": 0.8671875, + "rewards/format_reward/std": 0.33970388770103455, + "rewards/tag_count_reward/mean": 0.935546875, + "rewards/tag_count_reward/std": 0.18238498270511627, + "step": 2927 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.05078125, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 2013.0, + "completions/mean_length": 837.9765625, + "completions/mean_terminated_length": 773.2427978515625, + "completions/min_length": 93.0, + "completions/min_terminated_length": 93.0, + "epoch": 0.9995732696082615, + "grad_norm": 0.9258942604064941, + "kl": 8.15625, + "learning_rate": 1.0000127738493448e-07, + "loss": 0.5488, + "num_tokens": 1548449850.0, + "reward": 1.81396484375, + "reward_std": 0.5616399049758911, + "rewards/accuracy_reward/mean": 0.07056451588869095, + "rewards/accuracy_reward/std": 0.25635457038879395, + "rewards/format_reward/mean": 0.837890625, + "rewards/format_reward/std": 0.3689115643501282, + "rewards/tag_count_reward/mean": 0.90771484375, + "rewards/tag_count_reward/std": 0.22427105903625488, + "step": 2928 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": 0.027027027027026973, + "completions/max_length": 2048.0, + "completions/max_terminated_length": 1615.0, + "completions/mean_length": 800.9459838867188, + "completions/mean_terminated_length": 766.3055419921875, + "completions/min_length": 272.0, + "completions/min_terminated_length": 272.0, + "epoch": 0.9999146539216524, + "grad_norm": 1.447707176208496, + "kl": 5.875, + "learning_rate": 1.0000031934634693e-07, + "loss": 0.3851, + "num_tokens": 1548957970.0, + "reward": 1.91943359375, + "reward_std": 0.47769179940223694, + "rewards/accuracy_reward/mean": 0.091796875, + "rewards/accuracy_reward/std": 0.289021372795105, + "rewards/format_reward/mean": 0.88671875, + "rewards/format_reward/std": 0.3172462284564972, + "rewards/tag_count_reward/mean": 0.94091796875, + "rewards/tag_count_reward/std": 0.17639723420143127, + "step": 2929 + }, + { + "epoch": 0.9999146539216524, + "step": 2929, + "total_flos": 0.0, + "train_loss": 0.5993422349777706, + "train_runtime": 77858.5267, + "train_samples_per_second": 1.204, + "train_steps_per_second": 0.038 + } + ], + "logging_steps": 1, + "max_steps": 2929, + "num_input_tokens_seen": 1548957970, + "num_train_epochs": 1, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +}