| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.4644412191582003, | |
| "eval_steps": 500, | |
| "global_step": 400, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "completion_length": 724.58984375, | |
| "epoch": 0.0011611030478955006, | |
| "grad_norm": 0.7112110585443041, | |
| "kl": 0.0009012222290039062, | |
| "learning_rate": 3.846153846153846e-08, | |
| "loss": 0.0, | |
| "reward": 0.046875, | |
| "reward_std": 0.06733439117670059, | |
| "rewards/correctness_reward_func": 0.0390625, | |
| "rewards/strict_format_reward_func": 0.0078125, | |
| "step": 1 | |
| }, | |
| { | |
| "completion_length": 708.56640625, | |
| "epoch": 0.0023222060957910013, | |
| "grad_norm": 0.12068406687948798, | |
| "kl": 0.000453948974609375, | |
| "learning_rate": 7.692307692307692e-08, | |
| "loss": 0.0, | |
| "reward": 0.033203125, | |
| "reward_std": 0.06640625, | |
| "rewards/correctness_reward_func": 0.03125, | |
| "rewards/strict_format_reward_func": 0.001953125, | |
| "step": 2 | |
| }, | |
| { | |
| "completion_length": 742.65625, | |
| "epoch": 0.0034833091436865023, | |
| "grad_norm": 0.7976659386050398, | |
| "kl": 0.0007343292236328125, | |
| "learning_rate": 1.1538461538461539e-07, | |
| "loss": 0.0, | |
| "reward": 0.0234375, | |
| "reward_std": 0.046875, | |
| "rewards/correctness_reward_func": 0.015625, | |
| "rewards/strict_format_reward_func": 0.0078125, | |
| "step": 3 | |
| }, | |
| { | |
| "completion_length": 808.21875, | |
| "epoch": 0.0046444121915820025, | |
| "grad_norm": 0.40112277888275455, | |
| "kl": 0.00063323974609375, | |
| "learning_rate": 1.5384615384615385e-07, | |
| "loss": 0.0, | |
| "reward": 0.060546875, | |
| "reward_std": 0.1078859455883503, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.005859375, | |
| "step": 4 | |
| }, | |
| { | |
| "completion_length": 654.234375, | |
| "epoch": 0.005805515239477504, | |
| "grad_norm": 0.39725912723012635, | |
| "kl": 0.0004980564117431641, | |
| "learning_rate": 1.9230769230769231e-07, | |
| "loss": 0.0, | |
| "reward": 0.044921875, | |
| "reward_std": 0.08984375, | |
| "rewards/correctness_reward_func": 0.03125, | |
| "rewards/strict_format_reward_func": 0.013671875, | |
| "step": 5 | |
| }, | |
| { | |
| "completion_length": 772.68359375, | |
| "epoch": 0.006966618287373005, | |
| "grad_norm": 0.14294736924830026, | |
| "kl": 0.0004153251647949219, | |
| "learning_rate": 2.3076923076923078e-07, | |
| "loss": 0.0, | |
| "reward": 0.046875, | |
| "reward_std": 0.0805421955883503, | |
| "rewards/correctness_reward_func": 0.0390625, | |
| "rewards/strict_format_reward_func": 0.0078125, | |
| "step": 6 | |
| }, | |
| { | |
| "completion_length": 745.203125, | |
| "epoch": 0.008127721335268505, | |
| "grad_norm": 0.16815060965909404, | |
| "kl": 0.0009188652038574219, | |
| "learning_rate": 2.692307692307692e-07, | |
| "loss": 0.0, | |
| "reward": 0.021484375, | |
| "reward_std": 0.029760945588350296, | |
| "rewards/correctness_reward_func": 0.015625, | |
| "rewards/strict_format_reward_func": 0.005859375, | |
| "step": 7 | |
| }, | |
| { | |
| "completion_length": 744.265625, | |
| "epoch": 0.009288824383164005, | |
| "grad_norm": 12.779442066715234, | |
| "kl": 0.00357818603515625, | |
| "learning_rate": 3.076923076923077e-07, | |
| "loss": 0.0001, | |
| "reward": 0.03125, | |
| "reward_std": 0.049292195588350296, | |
| "rewards/correctness_reward_func": 0.03125, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "step": 8 | |
| }, | |
| { | |
| "completion_length": 704.51953125, | |
| "epoch": 0.010449927431059507, | |
| "grad_norm": 0.2475578183097461, | |
| "kl": 0.0035419464111328125, | |
| "learning_rate": 3.461538461538461e-07, | |
| "loss": 0.0001, | |
| "reward": 0.060546875, | |
| "reward_std": 0.1078859455883503, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.005859375, | |
| "step": 9 | |
| }, | |
| { | |
| "completion_length": 759.2890625, | |
| "epoch": 0.011611030478955007, | |
| "grad_norm": 0.3357083329970443, | |
| "kl": 0.00310516357421875, | |
| "learning_rate": 3.8461538461538463e-07, | |
| "loss": 0.0001, | |
| "reward": 0.00390625, | |
| "reward_std": 0.0078125, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.00390625, | |
| "step": 10 | |
| }, | |
| { | |
| "completion_length": 746.0703125, | |
| "epoch": 0.012772133526850507, | |
| "grad_norm": 0.2657165785401948, | |
| "kl": 0.00334930419921875, | |
| "learning_rate": 4.2307692307692304e-07, | |
| "loss": 0.0001, | |
| "reward": 0.017578125, | |
| "reward_std": 0.03515625, | |
| "rewards/correctness_reward_func": 0.015625, | |
| "rewards/strict_format_reward_func": 0.001953125, | |
| "step": 11 | |
| }, | |
| { | |
| "completion_length": 702.21875, | |
| "epoch": 0.01393323657474601, | |
| "grad_norm": 1.2656634353615759, | |
| "kl": 0.0019159317016601562, | |
| "learning_rate": 4.6153846153846156e-07, | |
| "loss": 0.0001, | |
| "reward": 0.033203125, | |
| "reward_std": 0.06640625, | |
| "rewards/correctness_reward_func": 0.03125, | |
| "rewards/strict_format_reward_func": 0.001953125, | |
| "step": 12 | |
| }, | |
| { | |
| "completion_length": 735.78515625, | |
| "epoch": 0.01509433962264151, | |
| "grad_norm": 0.5357650576255997, | |
| "kl": 0.005596160888671875, | |
| "learning_rate": 5e-07, | |
| "loss": 0.0002, | |
| "reward": 0.06640625, | |
| "reward_std": 0.1280700732022524, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.01171875, | |
| "step": 13 | |
| }, | |
| { | |
| "completion_length": 802.51953125, | |
| "epoch": 0.01625544267053701, | |
| "grad_norm": 1.9685839556774372, | |
| "kl": 0.00266265869140625, | |
| "learning_rate": 5.384615384615384e-07, | |
| "loss": 0.0001, | |
| "reward": 0.03125, | |
| "reward_std": 0.059198048897087574, | |
| "rewards/correctness_reward_func": 0.015625, | |
| "rewards/strict_format_reward_func": 0.015625, | |
| "step": 14 | |
| }, | |
| { | |
| "completion_length": 702.47265625, | |
| "epoch": 0.01741654571843251, | |
| "grad_norm": 0.3022926592468717, | |
| "kl": 0.00206756591796875, | |
| "learning_rate": 5.769230769230768e-07, | |
| "loss": 0.0001, | |
| "reward": 0.03125, | |
| "reward_std": 0.049292195588350296, | |
| "rewards/correctness_reward_func": 0.0234375, | |
| "rewards/strict_format_reward_func": 0.0078125, | |
| "step": 15 | |
| }, | |
| { | |
| "completion_length": 714.55859375, | |
| "epoch": 0.01857764876632801, | |
| "grad_norm": 0.67295700611606, | |
| "kl": 0.006526947021484375, | |
| "learning_rate": 6.153846153846154e-07, | |
| "loss": 0.0003, | |
| "reward": 0.0546875, | |
| "reward_std": 0.04213257320225239, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.0078125, | |
| "step": 16 | |
| }, | |
| { | |
| "completion_length": 728.94921875, | |
| "epoch": 0.019738751814223514, | |
| "grad_norm": 0.12683035812623522, | |
| "kl": 0.003253936767578125, | |
| "learning_rate": 6.538461538461538e-07, | |
| "loss": 0.0001, | |
| "reward": 0.041015625, | |
| "reward_std": 0.0688234455883503, | |
| "rewards/correctness_reward_func": 0.03125, | |
| "rewards/strict_format_reward_func": 0.009765625, | |
| "step": 17 | |
| }, | |
| { | |
| "completion_length": 752.4609375, | |
| "epoch": 0.020899854862119014, | |
| "grad_norm": 0.7572918382078203, | |
| "kl": 0.002063751220703125, | |
| "learning_rate": 6.923076923076922e-07, | |
| "loss": 0.0001, | |
| "reward": 0.01953125, | |
| "reward_std": 0.0390625, | |
| "rewards/correctness_reward_func": 0.015625, | |
| "rewards/strict_format_reward_func": 0.00390625, | |
| "step": 18 | |
| }, | |
| { | |
| "completion_length": 686.39453125, | |
| "epoch": 0.022060957910014514, | |
| "grad_norm": 0.366311042586646, | |
| "kl": 0.00254058837890625, | |
| "learning_rate": 7.307692307692307e-07, | |
| "loss": 0.0001, | |
| "reward": 0.037109375, | |
| "reward_std": 0.07421875, | |
| "rewards/correctness_reward_func": 0.03125, | |
| "rewards/strict_format_reward_func": 0.005859375, | |
| "step": 19 | |
| }, | |
| { | |
| "completion_length": 814.58203125, | |
| "epoch": 0.023222060957910014, | |
| "grad_norm": 0.39037265038987773, | |
| "kl": 0.0024509429931640625, | |
| "learning_rate": 7.692307692307693e-07, | |
| "loss": 0.0001, | |
| "reward": 0.056640625, | |
| "reward_std": 0.1000734455883503, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.009765625, | |
| "step": 20 | |
| }, | |
| { | |
| "completion_length": 790.58203125, | |
| "epoch": 0.024383164005805515, | |
| "grad_norm": 0.16449028843577287, | |
| "kl": 0.0017910003662109375, | |
| "learning_rate": 8.076923076923077e-07, | |
| "loss": 0.0001, | |
| "reward": 0.021484375, | |
| "reward_std": 0.04296875, | |
| "rewards/correctness_reward_func": 0.015625, | |
| "rewards/strict_format_reward_func": 0.005859375, | |
| "step": 21 | |
| }, | |
| { | |
| "completion_length": 720.109375, | |
| "epoch": 0.025544267053701015, | |
| "grad_norm": 0.16050190810020273, | |
| "kl": 0.000865936279296875, | |
| "learning_rate": 8.461538461538461e-07, | |
| "loss": 0.0, | |
| "reward": 0.041015625, | |
| "reward_std": 0.08203125, | |
| "rewards/correctness_reward_func": 0.03125, | |
| "rewards/strict_format_reward_func": 0.009765625, | |
| "step": 22 | |
| }, | |
| { | |
| "completion_length": 772.921875, | |
| "epoch": 0.026705370101596515, | |
| "grad_norm": 1.022693227541973, | |
| "kl": 0.010603904724121094, | |
| "learning_rate": 8.846153846153846e-07, | |
| "loss": 0.0004, | |
| "reward": 0.0234375, | |
| "reward_std": 0.046875, | |
| "rewards/correctness_reward_func": 0.015625, | |
| "rewards/strict_format_reward_func": 0.0078125, | |
| "step": 23 | |
| }, | |
| { | |
| "completion_length": 785.9296875, | |
| "epoch": 0.02786647314949202, | |
| "grad_norm": 0.4975098781878154, | |
| "kl": 0.0006268024444580078, | |
| "learning_rate": 9.230769230769231e-07, | |
| "loss": 0.0, | |
| "reward": 0.0546875, | |
| "reward_std": 0.10463257133960724, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.0078125, | |
| "step": 24 | |
| }, | |
| { | |
| "completion_length": 808.91796875, | |
| "epoch": 0.02902757619738752, | |
| "grad_norm": 0.07636536312105723, | |
| "kl": 0.0006952285766601562, | |
| "learning_rate": 9.615384615384615e-07, | |
| "loss": 0.0, | |
| "reward": 0.03515625, | |
| "reward_std": 0.055702777579426765, | |
| "rewards/correctness_reward_func": 0.0234375, | |
| "rewards/strict_format_reward_func": 0.01171875, | |
| "step": 25 | |
| }, | |
| { | |
| "completion_length": 677.65234375, | |
| "epoch": 0.03018867924528302, | |
| "grad_norm": 105.92855708885655, | |
| "kl": 0.048847198486328125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.002, | |
| "reward": 0.033203125, | |
| "reward_std": 0.06640625, | |
| "rewards/correctness_reward_func": 0.015625, | |
| "rewards/strict_format_reward_func": 0.017578125, | |
| "step": 26 | |
| }, | |
| { | |
| "completion_length": 657.85546875, | |
| "epoch": 0.03134978229317852, | |
| "grad_norm": 0.0861458003290139, | |
| "kl": 0.004261970520019531, | |
| "learning_rate": 9.999964611162971e-07, | |
| "loss": 0.0002, | |
| "reward": 0.009765625, | |
| "reward_std": 0.01953125, | |
| "rewards/correctness_reward_func": 0.0078125, | |
| "rewards/strict_format_reward_func": 0.001953125, | |
| "step": 27 | |
| }, | |
| { | |
| "completion_length": 726.2421875, | |
| "epoch": 0.03251088534107402, | |
| "grad_norm": 0.14527394620380304, | |
| "kl": 0.00072479248046875, | |
| "learning_rate": 9.999858445152839e-07, | |
| "loss": 0.0, | |
| "reward": 0.033203125, | |
| "reward_std": 0.053198445588350296, | |
| "rewards/correctness_reward_func": 0.03125, | |
| "rewards/strict_format_reward_func": 0.001953125, | |
| "step": 28 | |
| }, | |
| { | |
| "completion_length": 646.53125, | |
| "epoch": 0.03367198838896952, | |
| "grad_norm": 0.4038815325919152, | |
| "kl": 0.008038520812988281, | |
| "learning_rate": 9.999681503472433e-07, | |
| "loss": 0.0003, | |
| "reward": 0.02734375, | |
| "reward_std": 0.043573048897087574, | |
| "rewards/correctness_reward_func": 0.015625, | |
| "rewards/strict_format_reward_func": 0.01171875, | |
| "step": 29 | |
| }, | |
| { | |
| "completion_length": 676.234375, | |
| "epoch": 0.03483309143686502, | |
| "grad_norm": 24.530460542475996, | |
| "kl": 0.08559513092041016, | |
| "learning_rate": 9.99943378862646e-07, | |
| "loss": 0.0034, | |
| "reward": 0.076171875, | |
| "reward_std": 0.1358339935541153, | |
| "rewards/correctness_reward_func": 0.0625, | |
| "rewards/strict_format_reward_func": 0.013671875, | |
| "step": 30 | |
| }, | |
| { | |
| "completion_length": 669.62890625, | |
| "epoch": 0.035994194484760524, | |
| "grad_norm": 12.948432597412872, | |
| "kl": 0.0016450881958007812, | |
| "learning_rate": 9.999115304121457e-07, | |
| "loss": 0.0001, | |
| "reward": 0.033203125, | |
| "reward_std": 0.06640625, | |
| "rewards/correctness_reward_func": 0.0234375, | |
| "rewards/strict_format_reward_func": 0.009765625, | |
| "step": 31 | |
| }, | |
| { | |
| "completion_length": 632.80078125, | |
| "epoch": 0.03715529753265602, | |
| "grad_norm": 0.14738843577567481, | |
| "kl": 0.0011644363403320312, | |
| "learning_rate": 9.998726054465744e-07, | |
| "loss": 0.0, | |
| "reward": 0.0546875, | |
| "reward_std": 0.106073047965765, | |
| "rewards/correctness_reward_func": 0.0390625, | |
| "rewards/strict_format_reward_func": 0.015625, | |
| "step": 32 | |
| }, | |
| { | |
| "completion_length": 686.90234375, | |
| "epoch": 0.038316400580551524, | |
| "grad_norm": 6.995419155886516, | |
| "kl": 0.056095123291015625, | |
| "learning_rate": 9.998266045169354e-07, | |
| "loss": 0.0022, | |
| "reward": 0.052734375, | |
| "reward_std": 0.0922609455883503, | |
| "rewards/correctness_reward_func": 0.0390625, | |
| "rewards/strict_format_reward_func": 0.013671875, | |
| "step": 33 | |
| }, | |
| { | |
| "completion_length": 655.90234375, | |
| "epoch": 0.03947750362844703, | |
| "grad_norm": 3.5735767009613224, | |
| "kl": 0.013265609741210938, | |
| "learning_rate": 9.997735282743968e-07, | |
| "loss": 0.0005, | |
| "reward": 0.076171875, | |
| "reward_std": 0.13439351692795753, | |
| "rewards/correctness_reward_func": 0.0625, | |
| "rewards/strict_format_reward_func": 0.013671875, | |
| "step": 34 | |
| }, | |
| { | |
| "completion_length": 639.82421875, | |
| "epoch": 0.040638606676342524, | |
| "grad_norm": 1.8475352137430245, | |
| "kl": 0.0025959014892578125, | |
| "learning_rate": 9.997133774702812e-07, | |
| "loss": 0.0001, | |
| "reward": 0.05078125, | |
| "reward_std": 0.09682007133960724, | |
| "rewards/correctness_reward_func": 0.03125, | |
| "rewards/strict_format_reward_func": 0.01953125, | |
| "step": 35 | |
| }, | |
| { | |
| "completion_length": 715.3125, | |
| "epoch": 0.04179970972423803, | |
| "grad_norm": 13.632018237363075, | |
| "kl": 0.21777725219726562, | |
| "learning_rate": 9.996461529560552e-07, | |
| "loss": 0.0087, | |
| "reward": 0.0390625, | |
| "reward_std": 0.078125, | |
| "rewards/correctness_reward_func": 0.0234375, | |
| "rewards/strict_format_reward_func": 0.015625, | |
| "step": 36 | |
| }, | |
| { | |
| "completion_length": 631.8203125, | |
| "epoch": 0.042960812772133525, | |
| "grad_norm": 10.015816840984483, | |
| "kl": 0.00942230224609375, | |
| "learning_rate": 9.995718556833178e-07, | |
| "loss": 0.0004, | |
| "reward": 0.072265625, | |
| "reward_std": 0.141229297965765, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.025390625, | |
| "step": 37 | |
| }, | |
| { | |
| "completion_length": 636.9140625, | |
| "epoch": 0.04412191582002903, | |
| "grad_norm": 0.18443450708075712, | |
| "kl": 0.001720428466796875, | |
| "learning_rate": 9.994904867037865e-07, | |
| "loss": 0.0001, | |
| "reward": 0.033203125, | |
| "reward_std": 0.046594543382525444, | |
| "rewards/correctness_reward_func": 0.015625, | |
| "rewards/strict_format_reward_func": 0.017578125, | |
| "step": 38 | |
| }, | |
| { | |
| "completion_length": 667.6484375, | |
| "epoch": 0.045283018867924525, | |
| "grad_norm": 0.7286107675191874, | |
| "kl": 0.027004241943359375, | |
| "learning_rate": 9.994020471692832e-07, | |
| "loss": 0.0011, | |
| "reward": 0.0625, | |
| "reward_std": 0.10044586285948753, | |
| "rewards/correctness_reward_func": 0.0390625, | |
| "rewards/strict_format_reward_func": 0.0234375, | |
| "step": 39 | |
| }, | |
| { | |
| "completion_length": 695.68359375, | |
| "epoch": 0.04644412191582003, | |
| "grad_norm": 0.7697487269128109, | |
| "kl": 0.002681732177734375, | |
| "learning_rate": 9.993065383317162e-07, | |
| "loss": 0.0001, | |
| "reward": 0.078125, | |
| "reward_std": 0.12407248839735985, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.0234375, | |
| "step": 40 | |
| }, | |
| { | |
| "completion_length": 678.390625, | |
| "epoch": 0.04760522496371553, | |
| "grad_norm": 0.33385332935798095, | |
| "kl": 0.0028896331787109375, | |
| "learning_rate": 9.992039615430648e-07, | |
| "loss": 0.0001, | |
| "reward": 0.06640625, | |
| "reward_std": 0.1137621309608221, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.01953125, | |
| "step": 41 | |
| }, | |
| { | |
| "completion_length": 660.2421875, | |
| "epoch": 0.04876632801161103, | |
| "grad_norm": 5.2234598338711615, | |
| "kl": 0.03473663330078125, | |
| "learning_rate": 9.990943182553578e-07, | |
| "loss": 0.0014, | |
| "reward": 0.05859375, | |
| "reward_std": 0.11058359593153, | |
| "rewards/correctness_reward_func": 0.0234375, | |
| "rewards/strict_format_reward_func": 0.03515625, | |
| "step": 42 | |
| }, | |
| { | |
| "completion_length": 599.66015625, | |
| "epoch": 0.04992743105950653, | |
| "grad_norm": 0.3125631745813204, | |
| "kl": 0.0029144287109375, | |
| "learning_rate": 9.989776100206547e-07, | |
| "loss": 0.0001, | |
| "reward": 0.087890625, | |
| "reward_std": 0.1592714935541153, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.033203125, | |
| "step": 43 | |
| }, | |
| { | |
| "completion_length": 695.41015625, | |
| "epoch": 0.05108853410740203, | |
| "grad_norm": 0.31129440427202076, | |
| "kl": 0.002468109130859375, | |
| "learning_rate": 9.98853838491023e-07, | |
| "loss": 0.0001, | |
| "reward": 0.046875, | |
| "reward_std": 0.08240367192775011, | |
| "rewards/correctness_reward_func": 0.0234375, | |
| "rewards/strict_format_reward_func": 0.0234375, | |
| "step": 44 | |
| }, | |
| { | |
| "completion_length": 594.67578125, | |
| "epoch": 0.05224963715529753, | |
| "grad_norm": 0.2962376329942141, | |
| "kl": 0.00382232666015625, | |
| "learning_rate": 9.98723005418515e-07, | |
| "loss": 0.0002, | |
| "reward": 0.091796875, | |
| "reward_std": 0.1253724191337824, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.044921875, | |
| "step": 45 | |
| }, | |
| { | |
| "completion_length": 672.73046875, | |
| "epoch": 0.05341074020319303, | |
| "grad_norm": 3.778170631942383, | |
| "kl": 0.077880859375, | |
| "learning_rate": 9.985851126551428e-07, | |
| "loss": 0.0031, | |
| "reward": 0.103515625, | |
| "reward_std": 0.1781556848436594, | |
| "rewards/correctness_reward_func": 0.0625, | |
| "rewards/strict_format_reward_func": 0.041015625, | |
| "step": 46 | |
| }, | |
| { | |
| "completion_length": 603.15625, | |
| "epoch": 0.054571843251088534, | |
| "grad_norm": 0.3069672598383264, | |
| "kl": 0.006290435791015625, | |
| "learning_rate": 9.98440162152852e-07, | |
| "loss": 0.0003, | |
| "reward": 0.08984375, | |
| "reward_std": 0.1551719233393669, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.04296875, | |
| "step": 47 | |
| }, | |
| { | |
| "completion_length": 632.2890625, | |
| "epoch": 0.05573294629898404, | |
| "grad_norm": 1.2667105724676624, | |
| "kl": 0.029102325439453125, | |
| "learning_rate": 9.982881559634946e-07, | |
| "loss": 0.0012, | |
| "reward": 0.08203125, | |
| "reward_std": 0.14755274169147015, | |
| "rewards/correctness_reward_func": 0.03125, | |
| "rewards/strict_format_reward_func": 0.05078125, | |
| "step": 48 | |
| }, | |
| { | |
| "completion_length": 571.69921875, | |
| "epoch": 0.056894049346879534, | |
| "grad_norm": 51.00112062221978, | |
| "kl": 0.9501419067382812, | |
| "learning_rate": 9.981290962387997e-07, | |
| "loss": 0.0379, | |
| "reward": 0.07421875, | |
| "reward_std": 0.12532384134829044, | |
| "rewards/correctness_reward_func": 0.015625, | |
| "rewards/strict_format_reward_func": 0.05859375, | |
| "step": 49 | |
| }, | |
| { | |
| "completion_length": 595.40234375, | |
| "epoch": 0.05805515239477504, | |
| "grad_norm": 1.04581231978298, | |
| "kl": 0.01190185546875, | |
| "learning_rate": 9.979629852303425e-07, | |
| "loss": 0.0005, | |
| "reward": 0.11328125, | |
| "reward_std": 0.18154333159327507, | |
| "rewards/correctness_reward_func": 0.03125, | |
| "rewards/strict_format_reward_func": 0.08203125, | |
| "step": 50 | |
| }, | |
| { | |
| "completion_length": 561.47265625, | |
| "epoch": 0.059216255442670535, | |
| "grad_norm": 0.7109293653864751, | |
| "kl": 0.0103302001953125, | |
| "learning_rate": 9.977898252895134e-07, | |
| "loss": 0.0004, | |
| "reward": 0.13671875, | |
| "reward_std": 0.2260015867650509, | |
| "rewards/correctness_reward_func": 0.0390625, | |
| "rewards/strict_format_reward_func": 0.09765625, | |
| "step": 51 | |
| }, | |
| { | |
| "completion_length": 589.95703125, | |
| "epoch": 0.06037735849056604, | |
| "grad_norm": 2.9062465124154735, | |
| "kl": 0.00753021240234375, | |
| "learning_rate": 9.976096188674836e-07, | |
| "loss": 0.0003, | |
| "reward": 0.12890625, | |
| "reward_std": 0.19242635369300842, | |
| "rewards/correctness_reward_func": 0.0234375, | |
| "rewards/strict_format_reward_func": 0.10546875, | |
| "step": 52 | |
| }, | |
| { | |
| "completion_length": 537.0234375, | |
| "epoch": 0.06153846153846154, | |
| "grad_norm": 22.409980149466673, | |
| "kl": 0.2718505859375, | |
| "learning_rate": 9.974223685151718e-07, | |
| "loss": 0.0109, | |
| "reward": 0.12890625, | |
| "reward_std": 0.1836371347308159, | |
| "rewards/correctness_reward_func": 0.0234375, | |
| "rewards/strict_format_reward_func": 0.10546875, | |
| "step": 53 | |
| }, | |
| { | |
| "completion_length": 531.80859375, | |
| "epoch": 0.06269956458635705, | |
| "grad_norm": 2.8826743442454297, | |
| "kl": 0.122955322265625, | |
| "learning_rate": 9.972280768832067e-07, | |
| "loss": 0.0049, | |
| "reward": 0.220703125, | |
| "reward_std": 0.3008614853024483, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.126953125, | |
| "step": 54 | |
| }, | |
| { | |
| "completion_length": 611.5, | |
| "epoch": 0.06386066763425254, | |
| "grad_norm": 2.352779269743852, | |
| "kl": 0.02130889892578125, | |
| "learning_rate": 9.970267467218902e-07, | |
| "loss": 0.0009, | |
| "reward": 0.232421875, | |
| "reward_std": 0.2863368093967438, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.177734375, | |
| "step": 55 | |
| }, | |
| { | |
| "completion_length": 555.91796875, | |
| "epoch": 0.06502177068214804, | |
| "grad_norm": 0.9680034082980018, | |
| "kl": 0.01513671875, | |
| "learning_rate": 9.968183808811585e-07, | |
| "loss": 0.0006, | |
| "reward": 0.24609375, | |
| "reward_std": 0.3278784677386284, | |
| "rewards/correctness_reward_func": 0.0703125, | |
| "rewards/strict_format_reward_func": 0.17578125, | |
| "step": 56 | |
| }, | |
| { | |
| "completion_length": 561.3203125, | |
| "epoch": 0.06618287373004354, | |
| "grad_norm": 5.155662797346911, | |
| "kl": 0.1178741455078125, | |
| "learning_rate": 9.966029823105415e-07, | |
| "loss": 0.0047, | |
| "reward": 0.291015625, | |
| "reward_std": 0.3391275480389595, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.197265625, | |
| "step": 57 | |
| }, | |
| { | |
| "completion_length": 574.9453125, | |
| "epoch": 0.06734397677793905, | |
| "grad_norm": 3.5240176122453306, | |
| "kl": 0.0246734619140625, | |
| "learning_rate": 9.96380554059121e-07, | |
| "loss": 0.001, | |
| "reward": 0.291015625, | |
| "reward_std": 0.31832827627658844, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.212890625, | |
| "step": 58 | |
| }, | |
| { | |
| "completion_length": 583.93359375, | |
| "epoch": 0.06850507982583454, | |
| "grad_norm": 3.558262716521494, | |
| "kl": 0.0449981689453125, | |
| "learning_rate": 9.961510992754882e-07, | |
| "loss": 0.0018, | |
| "reward": 0.28515625, | |
| "reward_std": 0.26169369369745255, | |
| "rewards/correctness_reward_func": 0.0703125, | |
| "rewards/strict_format_reward_func": 0.21484375, | |
| "step": 59 | |
| }, | |
| { | |
| "completion_length": 498.65625, | |
| "epoch": 0.06966618287373004, | |
| "grad_norm": 2.811935806633887, | |
| "kl": 0.0194091796875, | |
| "learning_rate": 9.959146212076978e-07, | |
| "loss": 0.0008, | |
| "reward": 0.33984375, | |
| "reward_std": 0.314444862306118, | |
| "rewards/correctness_reward_func": 0.0625, | |
| "rewards/strict_format_reward_func": 0.27734375, | |
| "step": 60 | |
| }, | |
| { | |
| "completion_length": 478.53125, | |
| "epoch": 0.07082728592162554, | |
| "grad_norm": 4.984861468141312, | |
| "kl": 0.107757568359375, | |
| "learning_rate": 9.95671123203224e-07, | |
| "loss": 0.0043, | |
| "reward": 0.361328125, | |
| "reward_std": 0.33764002099633217, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.267578125, | |
| "step": 61 | |
| }, | |
| { | |
| "completion_length": 512.39453125, | |
| "epoch": 0.07198838896952105, | |
| "grad_norm": 1.007778929339388, | |
| "kl": 0.01971435546875, | |
| "learning_rate": 9.954206087089105e-07, | |
| "loss": 0.0008, | |
| "reward": 0.3203125, | |
| "reward_std": 0.2790137939155102, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.2734375, | |
| "step": 62 | |
| }, | |
| { | |
| "completion_length": 516.03515625, | |
| "epoch": 0.07314949201741655, | |
| "grad_norm": 0.5960945531101304, | |
| "kl": 0.0254974365234375, | |
| "learning_rate": 9.951630812709244e-07, | |
| "loss": 0.001, | |
| "reward": 0.373046875, | |
| "reward_std": 0.3191210627555847, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.279296875, | |
| "step": 63 | |
| }, | |
| { | |
| "completion_length": 478.65625, | |
| "epoch": 0.07431059506531204, | |
| "grad_norm": 3.3120058314917094, | |
| "kl": 0.027435302734375, | |
| "learning_rate": 9.948985445347044e-07, | |
| "loss": 0.0011, | |
| "reward": 0.396484375, | |
| "reward_std": 0.30096181109547615, | |
| "rewards/correctness_reward_func": 0.0703125, | |
| "rewards/strict_format_reward_func": 0.326171875, | |
| "step": 64 | |
| }, | |
| { | |
| "completion_length": 453.5390625, | |
| "epoch": 0.07547169811320754, | |
| "grad_norm": 10.565800054920043, | |
| "kl": 0.06463623046875, | |
| "learning_rate": 9.946270022449093e-07, | |
| "loss": 0.0026, | |
| "reward": 0.40234375, | |
| "reward_std": 0.30288316309452057, | |
| "rewards/correctness_reward_func": 0.0703125, | |
| "rewards/strict_format_reward_func": 0.33203125, | |
| "step": 65 | |
| }, | |
| { | |
| "completion_length": 467.4609375, | |
| "epoch": 0.07663280116110305, | |
| "grad_norm": 8.848702328863897, | |
| "kl": 0.026763916015625, | |
| "learning_rate": 9.94348458245365e-07, | |
| "loss": 0.0011, | |
| "reward": 0.484375, | |
| "reward_std": 0.3539407253265381, | |
| "rewards/correctness_reward_func": 0.1640625, | |
| "rewards/strict_format_reward_func": 0.3203125, | |
| "step": 66 | |
| }, | |
| { | |
| "completion_length": 460.51171875, | |
| "epoch": 0.07779390420899855, | |
| "grad_norm": 0.8276997968349737, | |
| "kl": 0.028564453125, | |
| "learning_rate": 9.940629164790118e-07, | |
| "loss": 0.0011, | |
| "reward": 0.390625, | |
| "reward_std": 0.2685689851641655, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.3359375, | |
| "step": 67 | |
| }, | |
| { | |
| "completion_length": 432.30078125, | |
| "epoch": 0.07895500725689406, | |
| "grad_norm": 1.6501741381150759, | |
| "kl": 0.068115234375, | |
| "learning_rate": 9.937703809878454e-07, | |
| "loss": 0.0027, | |
| "reward": 0.447265625, | |
| "reward_std": 0.32472314685583115, | |
| "rewards/correctness_reward_func": 0.109375, | |
| "rewards/strict_format_reward_func": 0.337890625, | |
| "step": 68 | |
| }, | |
| { | |
| "completion_length": 433.5625, | |
| "epoch": 0.08011611030478955, | |
| "grad_norm": 5.548040313902584, | |
| "kl": 0.0538330078125, | |
| "learning_rate": 9.934708559128622e-07, | |
| "loss": 0.0022, | |
| "reward": 0.48046875, | |
| "reward_std": 0.3404357209801674, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.36328125, | |
| "step": 69 | |
| }, | |
| { | |
| "completion_length": 411.9921875, | |
| "epoch": 0.08127721335268505, | |
| "grad_norm": 4.54760002369526, | |
| "kl": 0.0399169921875, | |
| "learning_rate": 9.931643454939998e-07, | |
| "loss": 0.0016, | |
| "reward": 0.505859375, | |
| "reward_std": 0.3411124534904957, | |
| "rewards/correctness_reward_func": 0.1484375, | |
| "rewards/strict_format_reward_func": 0.357421875, | |
| "step": 70 | |
| }, | |
| { | |
| "completion_length": 374.98828125, | |
| "epoch": 0.08243831640058055, | |
| "grad_norm": 0.7476462962662087, | |
| "kl": 0.06536865234375, | |
| "learning_rate": 9.928508540700772e-07, | |
| "loss": 0.0026, | |
| "reward": 0.458984375, | |
| "reward_std": 0.2434551902115345, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.365234375, | |
| "step": 71 | |
| }, | |
| { | |
| "completion_length": 371.80859375, | |
| "epoch": 0.08359941944847606, | |
| "grad_norm": 0.31697483250235303, | |
| "kl": 0.04022216796875, | |
| "learning_rate": 9.925303860787333e-07, | |
| "loss": 0.0016, | |
| "reward": 0.427734375, | |
| "reward_std": 0.24097520112991333, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.373046875, | |
| "step": 72 | |
| }, | |
| { | |
| "completion_length": 408.953125, | |
| "epoch": 0.08476052249637156, | |
| "grad_norm": 2.342541131973969, | |
| "kl": 0.0394287109375, | |
| "learning_rate": 9.92202946056364e-07, | |
| "loss": 0.0016, | |
| "reward": 0.447265625, | |
| "reward_std": 0.2658774182200432, | |
| "rewards/correctness_reward_func": 0.0625, | |
| "rewards/strict_format_reward_func": 0.384765625, | |
| "step": 73 | |
| }, | |
| { | |
| "completion_length": 367.03125, | |
| "epoch": 0.08592162554426705, | |
| "grad_norm": 0.457004333832563, | |
| "kl": 0.05194091796875, | |
| "learning_rate": 9.918685386380572e-07, | |
| "loss": 0.0021, | |
| "reward": 0.478515625, | |
| "reward_std": 0.22598736733198166, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.400390625, | |
| "step": 74 | |
| }, | |
| { | |
| "completion_length": 423.35546875, | |
| "epoch": 0.08708272859216255, | |
| "grad_norm": 81.83417853597544, | |
| "kl": 0.247314453125, | |
| "learning_rate": 9.915271685575296e-07, | |
| "loss": 0.0099, | |
| "reward": 0.48828125, | |
| "reward_std": 0.3054451234638691, | |
| "rewards/correctness_reward_func": 0.1015625, | |
| "rewards/strict_format_reward_func": 0.38671875, | |
| "step": 75 | |
| }, | |
| { | |
| "completion_length": 312.484375, | |
| "epoch": 0.08824383164005806, | |
| "grad_norm": 1.780578379981416, | |
| "kl": 0.0880126953125, | |
| "learning_rate": 9.911788406470568e-07, | |
| "loss": 0.0035, | |
| "reward": 0.548828125, | |
| "reward_std": 0.32934778556227684, | |
| "rewards/correctness_reward_func": 0.125, | |
| "rewards/strict_format_reward_func": 0.423828125, | |
| "step": 76 | |
| }, | |
| { | |
| "completion_length": 338.2421875, | |
| "epoch": 0.08940493468795356, | |
| "grad_norm": 0.6544200585420813, | |
| "kl": 0.0731201171875, | |
| "learning_rate": 9.908235598374068e-07, | |
| "loss": 0.0029, | |
| "reward": 0.45703125, | |
| "reward_std": 0.1958453767001629, | |
| "rewards/correctness_reward_func": 0.0390625, | |
| "rewards/strict_format_reward_func": 0.41796875, | |
| "step": 77 | |
| }, | |
| { | |
| "completion_length": 268.171875, | |
| "epoch": 0.09056603773584905, | |
| "grad_norm": 0.5879416696261314, | |
| "kl": 0.108642578125, | |
| "learning_rate": 9.904613311577695e-07, | |
| "loss": 0.0044, | |
| "reward": 0.556640625, | |
| "reward_std": 0.22046153992414474, | |
| "rewards/correctness_reward_func": 0.140625, | |
| "rewards/strict_format_reward_func": 0.416015625, | |
| "step": 78 | |
| }, | |
| { | |
| "completion_length": 231.109375, | |
| "epoch": 0.09172714078374455, | |
| "grad_norm": 0.8976017470783989, | |
| "kl": 0.18408203125, | |
| "learning_rate": 9.900921597356855e-07, | |
| "loss": 0.0074, | |
| "reward": 0.515625, | |
| "reward_std": 0.20618988201022148, | |
| "rewards/correctness_reward_func": 0.0859375, | |
| "rewards/strict_format_reward_func": 0.4296875, | |
| "step": 79 | |
| }, | |
| { | |
| "completion_length": 196.59765625, | |
| "epoch": 0.09288824383164006, | |
| "grad_norm": 17.77907159035059, | |
| "kl": 0.263916015625, | |
| "learning_rate": 9.897160507969735e-07, | |
| "loss": 0.0106, | |
| "reward": 0.47265625, | |
| "reward_std": 0.19209052622318268, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.41796875, | |
| "step": 80 | |
| }, | |
| { | |
| "completion_length": 110.4609375, | |
| "epoch": 0.09404934687953556, | |
| "grad_norm": 1.3276411102732406, | |
| "kl": 0.3662109375, | |
| "learning_rate": 9.893330096656573e-07, | |
| "loss": 0.0147, | |
| "reward": 0.578125, | |
| "reward_std": 0.27833693847060204, | |
| "rewards/correctness_reward_func": 0.140625, | |
| "rewards/strict_format_reward_func": 0.4375, | |
| "step": 81 | |
| }, | |
| { | |
| "completion_length": 77.88671875, | |
| "epoch": 0.09521044992743107, | |
| "grad_norm": 1.9379785150930775, | |
| "kl": 0.51953125, | |
| "learning_rate": 9.889430417638883e-07, | |
| "loss": 0.0208, | |
| "reward": 0.51171875, | |
| "reward_std": 0.1544804871082306, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.45703125, | |
| "step": 82 | |
| }, | |
| { | |
| "completion_length": 73.9375, | |
| "epoch": 0.09637155297532655, | |
| "grad_norm": 0.8473473359273695, | |
| "kl": 0.48486328125, | |
| "learning_rate": 9.885461526118713e-07, | |
| "loss": 0.0194, | |
| "reward": 0.50390625, | |
| "reward_std": 0.14182795584201813, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.45703125, | |
| "step": 83 | |
| }, | |
| { | |
| "completion_length": 56.2109375, | |
| "epoch": 0.09753265602322206, | |
| "grad_norm": 2.281085454498393, | |
| "kl": 0.49072265625, | |
| "learning_rate": 9.88142347827784e-07, | |
| "loss": 0.0196, | |
| "reward": 0.4765625, | |
| "reward_std": 0.12851098738610744, | |
| "rewards/correctness_reward_func": 0.0234375, | |
| "rewards/strict_format_reward_func": 0.453125, | |
| "step": 84 | |
| }, | |
| { | |
| "completion_length": 37.875, | |
| "epoch": 0.09869375907111756, | |
| "grad_norm": 0.5788019582854786, | |
| "kl": 0.5927734375, | |
| "learning_rate": 9.877316331276993e-07, | |
| "loss": 0.0237, | |
| "reward": 0.470703125, | |
| "reward_std": 0.055291797965765, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.470703125, | |
| "step": 85 | |
| }, | |
| { | |
| "completion_length": 33.50390625, | |
| "epoch": 0.09985486211901307, | |
| "grad_norm": 6.104259815482374, | |
| "kl": 0.5439453125, | |
| "learning_rate": 9.873140143255034e-07, | |
| "loss": 0.0217, | |
| "reward": 0.517578125, | |
| "reward_std": 0.07212539482861757, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.470703125, | |
| "step": 86 | |
| }, | |
| { | |
| "completion_length": 20.76171875, | |
| "epoch": 0.10101596516690857, | |
| "grad_norm": 0.507931311814518, | |
| "kl": 0.5947265625, | |
| "learning_rate": 9.86889497332814e-07, | |
| "loss": 0.0238, | |
| "reward": 0.486328125, | |
| "reward_std": 0.02734375, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.486328125, | |
| "step": 87 | |
| }, | |
| { | |
| "completion_length": 16.58203125, | |
| "epoch": 0.10217706821480406, | |
| "grad_norm": 0.45325431756004353, | |
| "kl": 0.6630859375, | |
| "learning_rate": 9.864580881588958e-07, | |
| "loss": 0.0265, | |
| "reward": 0.501953125, | |
| "reward_std": 0.024041798897087574, | |
| "rewards/correctness_reward_func": 0.0078125, | |
| "rewards/strict_format_reward_func": 0.494140625, | |
| "step": 88 | |
| }, | |
| { | |
| "completion_length": 16.12109375, | |
| "epoch": 0.10333817126269956, | |
| "grad_norm": 0.38014072113379394, | |
| "kl": 0.6513671875, | |
| "learning_rate": 9.860197929105767e-07, | |
| "loss": 0.0261, | |
| "reward": 0.49609375, | |
| "reward_std": 0.0078125, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.49609375, | |
| "step": 89 | |
| }, | |
| { | |
| "completion_length": 19.7265625, | |
| "epoch": 0.10449927431059507, | |
| "grad_norm": 3.246980620297076, | |
| "kl": 0.6064453125, | |
| "learning_rate": 9.8557461779216e-07, | |
| "loss": 0.0243, | |
| "reward": 0.48828125, | |
| "reward_std": 0.0234375, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.48828125, | |
| "step": 90 | |
| }, | |
| { | |
| "completion_length": 15.8359375, | |
| "epoch": 0.10566037735849057, | |
| "grad_norm": 1.125767741710113, | |
| "kl": 0.6162109375, | |
| "learning_rate": 9.85122569105338e-07, | |
| "loss": 0.0247, | |
| "reward": 0.49609375, | |
| "reward_std": 0.0078125, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.49609375, | |
| "step": 91 | |
| }, | |
| { | |
| "completion_length": 22.44921875, | |
| "epoch": 0.10682148040638606, | |
| "grad_norm": 0.44874987843781083, | |
| "kl": 0.6083984375, | |
| "learning_rate": 9.846636532491012e-07, | |
| "loss": 0.0243, | |
| "reward": 0.48828125, | |
| "reward_std": 0.0234375, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.48828125, | |
| "step": 92 | |
| }, | |
| { | |
| "completion_length": 16.21484375, | |
| "epoch": 0.10798258345428156, | |
| "grad_norm": 2.7244778647827363, | |
| "kl": 0.796875, | |
| "learning_rate": 9.841978767196493e-07, | |
| "loss": 0.0319, | |
| "reward": 0.494140625, | |
| "reward_std": 0.01171875, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.494140625, | |
| "step": 93 | |
| }, | |
| { | |
| "completion_length": 18.08984375, | |
| "epoch": 0.10914368650217707, | |
| "grad_norm": 0.900717061581606, | |
| "kl": 0.5712890625, | |
| "learning_rate": 9.83725246110298e-07, | |
| "loss": 0.0229, | |
| "reward": 0.49609375, | |
| "reward_std": 0.0234375, | |
| "rewards/correctness_reward_func": 0.0078125, | |
| "rewards/strict_format_reward_func": 0.48828125, | |
| "step": 94 | |
| }, | |
| { | |
| "completion_length": 19.921875, | |
| "epoch": 0.11030478955007257, | |
| "grad_norm": 0.2922094758318377, | |
| "kl": 0.62109375, | |
| "learning_rate": 9.832457681113865e-07, | |
| "loss": 0.0249, | |
| "reward": 0.51953125, | |
| "reward_std": 0.0546875, | |
| "rewards/correctness_reward_func": 0.03125, | |
| "rewards/strict_format_reward_func": 0.48828125, | |
| "step": 95 | |
| }, | |
| { | |
| "completion_length": 22.48828125, | |
| "epoch": 0.11146589259796807, | |
| "grad_norm": 1.5719055280514225, | |
| "kl": 0.5966796875, | |
| "learning_rate": 9.827594495101822e-07, | |
| "loss": 0.0239, | |
| "reward": 0.53125, | |
| "reward_std": 0.049292195588350296, | |
| "rewards/correctness_reward_func": 0.0390625, | |
| "rewards/strict_format_reward_func": 0.4921875, | |
| "step": 96 | |
| }, | |
| { | |
| "completion_length": 21.078125, | |
| "epoch": 0.11262699564586356, | |
| "grad_norm": 1.875138112811585, | |
| "kl": 0.62890625, | |
| "learning_rate": 9.822662971907852e-07, | |
| "loss": 0.0252, | |
| "reward": 0.505859375, | |
| "reward_std": 0.047479298897087574, | |
| "rewards/correctness_reward_func": 0.015625, | |
| "rewards/strict_format_reward_func": 0.490234375, | |
| "step": 97 | |
| }, | |
| { | |
| "completion_length": 20.328125, | |
| "epoch": 0.11378809869375907, | |
| "grad_norm": 0.46440948422543626, | |
| "kl": 0.5703125, | |
| "learning_rate": 9.8176631813403e-07, | |
| "loss": 0.0228, | |
| "reward": 0.5078125, | |
| "reward_std": 0.015625, | |
| "rewards/correctness_reward_func": 0.0078125, | |
| "rewards/strict_format_reward_func": 0.5, | |
| "step": 98 | |
| }, | |
| { | |
| "completion_length": 17.6953125, | |
| "epoch": 0.11494920174165457, | |
| "grad_norm": 2.350666772323524, | |
| "kl": 0.6298828125, | |
| "learning_rate": 9.812595194173874e-07, | |
| "loss": 0.0252, | |
| "reward": 0.50390625, | |
| "reward_std": 0.0234375, | |
| "rewards/correctness_reward_func": 0.0078125, | |
| "rewards/strict_format_reward_func": 0.49609375, | |
| "step": 99 | |
| }, | |
| { | |
| "completion_length": 30.88671875, | |
| "epoch": 0.11611030478955008, | |
| "grad_norm": 0.6769899791454503, | |
| "kl": 0.537109375, | |
| "learning_rate": 9.807459082148648e-07, | |
| "loss": 0.0215, | |
| "reward": 0.56640625, | |
| "reward_std": 0.0804273895919323, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.48828125, | |
| "step": 100 | |
| }, | |
| { | |
| "completion_length": 38.8203125, | |
| "epoch": 0.11727140783744558, | |
| "grad_norm": 9.613286964921125, | |
| "kl": 0.53515625, | |
| "learning_rate": 9.802254917969032e-07, | |
| "loss": 0.0214, | |
| "reward": 0.5390625, | |
| "reward_std": 0.1328125, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 101 | |
| }, | |
| { | |
| "completion_length": 21.05078125, | |
| "epoch": 0.11843251088534107, | |
| "grad_norm": 0.383590636095333, | |
| "kl": 0.5419921875, | |
| "learning_rate": 9.796982775302754e-07, | |
| "loss": 0.0217, | |
| "reward": 0.52734375, | |
| "reward_std": 0.057104695588350296, | |
| "rewards/correctness_reward_func": 0.03125, | |
| "rewards/strict_format_reward_func": 0.49609375, | |
| "step": 102 | |
| }, | |
| { | |
| "completion_length": 26.5234375, | |
| "epoch": 0.11959361393323657, | |
| "grad_norm": 0.45036244814830856, | |
| "kl": 0.53759765625, | |
| "learning_rate": 9.79164272877981e-07, | |
| "loss": 0.0215, | |
| "reward": 0.53125, | |
| "reward_std": 0.090448047965765, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 103 | |
| }, | |
| { | |
| "completion_length": 23.54296875, | |
| "epoch": 0.12075471698113208, | |
| "grad_norm": 2.2388808963947713, | |
| "kl": 0.5615234375, | |
| "learning_rate": 9.786234853991418e-07, | |
| "loss": 0.0224, | |
| "reward": 0.56640625, | |
| "reward_std": 0.08606802299618721, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.48828125, | |
| "step": 104 | |
| }, | |
| { | |
| "completion_length": 53.6875, | |
| "epoch": 0.12191582002902758, | |
| "grad_norm": 1.3962986009964624, | |
| "kl": 0.515625, | |
| "learning_rate": 9.780759227488937e-07, | |
| "loss": 0.0207, | |
| "reward": 0.55859375, | |
| "reward_std": 0.1321367770433426, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 105 | |
| }, | |
| { | |
| "completion_length": 45.9140625, | |
| "epoch": 0.12307692307692308, | |
| "grad_norm": 0.6991087377799369, | |
| "kl": 0.5751953125, | |
| "learning_rate": 9.775215926782787e-07, | |
| "loss": 0.023, | |
| "reward": 0.5390625, | |
| "reward_std": 0.12271332554519176, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 106 | |
| }, | |
| { | |
| "completion_length": 41.71484375, | |
| "epoch": 0.12423802612481857, | |
| "grad_norm": 1.113559415990131, | |
| "kl": 0.501953125, | |
| "learning_rate": 9.769605030341355e-07, | |
| "loss": 0.0201, | |
| "reward": 0.494140625, | |
| "reward_std": 0.039666798897087574, | |
| "rewards/correctness_reward_func": 0.0078125, | |
| "rewards/strict_format_reward_func": 0.486328125, | |
| "step": 107 | |
| }, | |
| { | |
| "completion_length": 30.234375, | |
| "epoch": 0.1253991291727141, | |
| "grad_norm": 9.444282741370284, | |
| "kl": 0.52587890625, | |
| "learning_rate": 9.763926617589884e-07, | |
| "loss": 0.021, | |
| "reward": 0.529296875, | |
| "reward_std": 0.09918868914246559, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.482421875, | |
| "step": 108 | |
| }, | |
| { | |
| "completion_length": 36.328125, | |
| "epoch": 0.12656023222060958, | |
| "grad_norm": 0.5876435788826965, | |
| "kl": 0.49365234375, | |
| "learning_rate": 9.758180768909336e-07, | |
| "loss": 0.0198, | |
| "reward": 0.5625, | |
| "reward_std": 0.1117921955883503, | |
| "rewards/correctness_reward_func": 0.0703125, | |
| "rewards/strict_format_reward_func": 0.4921875, | |
| "step": 109 | |
| }, | |
| { | |
| "completion_length": 45.109375, | |
| "epoch": 0.12772133526850507, | |
| "grad_norm": 0.6967401340285807, | |
| "kl": 0.58203125, | |
| "learning_rate": 9.75236756563528e-07, | |
| "loss": 0.0233, | |
| "reward": 0.564453125, | |
| "reward_std": 0.1313234455883503, | |
| "rewards/correctness_reward_func": 0.0859375, | |
| "rewards/strict_format_reward_func": 0.478515625, | |
| "step": 110 | |
| }, | |
| { | |
| "completion_length": 22.265625, | |
| "epoch": 0.1288824383164006, | |
| "grad_norm": 4.157806679866693, | |
| "kl": 0.5859375, | |
| "learning_rate": 9.746487090056712e-07, | |
| "loss": 0.0235, | |
| "reward": 0.57421875, | |
| "reward_std": 0.1376468911767006, | |
| "rewards/correctness_reward_func": 0.0859375, | |
| "rewards/strict_format_reward_func": 0.48828125, | |
| "step": 111 | |
| }, | |
| { | |
| "completion_length": 43.33203125, | |
| "epoch": 0.13004354136429608, | |
| "grad_norm": 1.3608549571109825, | |
| "kl": 0.52099609375, | |
| "learning_rate": 9.740539425414911e-07, | |
| "loss": 0.0208, | |
| "reward": 0.548828125, | |
| "reward_std": 0.1078859455883503, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.494140625, | |
| "step": 112 | |
| }, | |
| { | |
| "completion_length": 27.4609375, | |
| "epoch": 0.1312046444121916, | |
| "grad_norm": 0.9303195344407332, | |
| "kl": 0.5302734375, | |
| "learning_rate": 9.734524655902252e-07, | |
| "loss": 0.0212, | |
| "reward": 0.5, | |
| "reward_std": 0.03125, | |
| "rewards/correctness_reward_func": 0.0078125, | |
| "rewards/strict_format_reward_func": 0.4921875, | |
| "step": 113 | |
| }, | |
| { | |
| "completion_length": 44.421875, | |
| "epoch": 0.13236574746008709, | |
| "grad_norm": 2.5989058419028512, | |
| "kl": 0.5224609375, | |
| "learning_rate": 9.728442866661012e-07, | |
| "loss": 0.0209, | |
| "reward": 0.5390625, | |
| "reward_std": 0.09286524448543787, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 114 | |
| }, | |
| { | |
| "completion_length": 55.43359375, | |
| "epoch": 0.13352685050798258, | |
| "grad_norm": 4.35900633763163, | |
| "kl": 0.513671875, | |
| "learning_rate": 9.722294143782171e-07, | |
| "loss": 0.0205, | |
| "reward": 0.5234375, | |
| "reward_std": 0.09476527757942677, | |
| "rewards/correctness_reward_func": 0.0390625, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 115 | |
| }, | |
| { | |
| "completion_length": 32.14453125, | |
| "epoch": 0.1346879535558781, | |
| "grad_norm": 2.5576115813189664, | |
| "kl": 0.48486328125, | |
| "learning_rate": 9.71607857430419e-07, | |
| "loss": 0.0194, | |
| "reward": 0.548828125, | |
| "reward_std": 0.1358339935541153, | |
| "rewards/correctness_reward_func": 0.0625, | |
| "rewards/strict_format_reward_func": 0.486328125, | |
| "step": 116 | |
| }, | |
| { | |
| "completion_length": 40.48828125, | |
| "epoch": 0.13584905660377358, | |
| "grad_norm": 5.3055382808430025, | |
| "kl": 0.55078125, | |
| "learning_rate": 9.709796246211776e-07, | |
| "loss": 0.022, | |
| "reward": 0.599609375, | |
| "reward_std": 0.1770055592060089, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.482421875, | |
| "step": 117 | |
| }, | |
| { | |
| "completion_length": 48.4296875, | |
| "epoch": 0.13701015965166907, | |
| "grad_norm": 1.9357783363747145, | |
| "kl": 0.52734375, | |
| "learning_rate": 9.70344724843465e-07, | |
| "loss": 0.0211, | |
| "reward": 0.623046875, | |
| "reward_std": 0.0844484455883503, | |
| "rewards/correctness_reward_func": 0.1328125, | |
| "rewards/strict_format_reward_func": 0.490234375, | |
| "step": 118 | |
| }, | |
| { | |
| "completion_length": 33.06640625, | |
| "epoch": 0.1381712626995646, | |
| "grad_norm": 0.9093609363691819, | |
| "kl": 0.48876953125, | |
| "learning_rate": 9.697031670846264e-07, | |
| "loss": 0.0195, | |
| "reward": 0.595703125, | |
| "reward_std": 0.1571781411767006, | |
| "rewards/correctness_reward_func": 0.109375, | |
| "rewards/strict_format_reward_func": 0.486328125, | |
| "step": 119 | |
| }, | |
| { | |
| "completion_length": 26.15234375, | |
| "epoch": 0.13933236574746008, | |
| "grad_norm": 1.2956378319464605, | |
| "kl": 0.52490234375, | |
| "learning_rate": 9.690549604262554e-07, | |
| "loss": 0.021, | |
| "reward": 0.611328125, | |
| "reward_std": 0.1670839935541153, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.494140625, | |
| "step": 120 | |
| }, | |
| { | |
| "completion_length": 23.31640625, | |
| "epoch": 0.1404934687953556, | |
| "grad_norm": 1.0404878413954166, | |
| "kl": 0.53173828125, | |
| "learning_rate": 9.684001140440638e-07, | |
| "loss": 0.0213, | |
| "reward": 0.533203125, | |
| "reward_std": 0.08553362637758255, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.486328125, | |
| "step": 121 | |
| }, | |
| { | |
| "completion_length": 33.63671875, | |
| "epoch": 0.1416545718432511, | |
| "grad_norm": 0.5491271196977439, | |
| "kl": 0.5263671875, | |
| "learning_rate": 9.677386372077523e-07, | |
| "loss": 0.0211, | |
| "reward": 0.537109375, | |
| "reward_std": 0.0798230916261673, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.490234375, | |
| "step": 122 | |
| }, | |
| { | |
| "completion_length": 27.7578125, | |
| "epoch": 0.14281567489114658, | |
| "grad_norm": 1.4086546997823266, | |
| "kl": 0.49462890625, | |
| "learning_rate": 9.670705392808795e-07, | |
| "loss": 0.0198, | |
| "reward": 0.642578125, | |
| "reward_std": 0.14176217839121819, | |
| "rewards/correctness_reward_func": 0.15625, | |
| "rewards/strict_format_reward_func": 0.486328125, | |
| "step": 123 | |
| }, | |
| { | |
| "completion_length": 46.31640625, | |
| "epoch": 0.1439767779390421, | |
| "grad_norm": 0.44935314029572543, | |
| "kl": 0.4541015625, | |
| "learning_rate": 9.663958297207285e-07, | |
| "loss": 0.0181, | |
| "reward": 0.5390625, | |
| "reward_std": 0.12601527571678162, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 124 | |
| }, | |
| { | |
| "completion_length": 42.33203125, | |
| "epoch": 0.14513788098693758, | |
| "grad_norm": 4.134753027574554, | |
| "kl": 0.45703125, | |
| "learning_rate": 9.657145180781742e-07, | |
| "loss": 0.0183, | |
| "reward": 0.513671875, | |
| "reward_std": 0.06089614052325487, | |
| "rewards/correctness_reward_func": 0.0234375, | |
| "rewards/strict_format_reward_func": 0.490234375, | |
| "step": 125 | |
| }, | |
| { | |
| "completion_length": 42.03515625, | |
| "epoch": 0.1462989840348331, | |
| "grad_norm": 0.4544547991850145, | |
| "kl": 0.40625, | |
| "learning_rate": 9.650266139975473e-07, | |
| "loss": 0.0163, | |
| "reward": 0.5703125, | |
| "reward_std": 0.12411524076014757, | |
| "rewards/correctness_reward_func": 0.0859375, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 126 | |
| }, | |
| { | |
| "completion_length": 31.16796875, | |
| "epoch": 0.1474600870827286, | |
| "grad_norm": 1.6260031691388108, | |
| "kl": 0.486328125, | |
| "learning_rate": 9.64332127216498e-07, | |
| "loss": 0.0194, | |
| "reward": 0.51171875, | |
| "reward_std": 0.0390625, | |
| "rewards/correctness_reward_func": 0.0234375, | |
| "rewards/strict_format_reward_func": 0.48828125, | |
| "step": 127 | |
| }, | |
| { | |
| "completion_length": 32.0, | |
| "epoch": 0.14862119013062408, | |
| "grad_norm": 0.5382173674869626, | |
| "kl": 0.4267578125, | |
| "learning_rate": 9.63631067565858e-07, | |
| "loss": 0.0171, | |
| "reward": 0.529296875, | |
| "reward_std": 0.053198445588350296, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.482421875, | |
| "step": 128 | |
| }, | |
| { | |
| "completion_length": 44.75390625, | |
| "epoch": 0.1497822931785196, | |
| "grad_norm": 0.7683808964372313, | |
| "kl": 0.4228515625, | |
| "learning_rate": 9.629234449695015e-07, | |
| "loss": 0.0169, | |
| "reward": 0.5625, | |
| "reward_std": 0.1478765867650509, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 129 | |
| }, | |
| { | |
| "completion_length": 40.52734375, | |
| "epoch": 0.1509433962264151, | |
| "grad_norm": 1.0117014376716735, | |
| "kl": 0.45263671875, | |
| "learning_rate": 9.62209269444205e-07, | |
| "loss": 0.0181, | |
| "reward": 0.65625, | |
| "reward_std": 0.218528650701046, | |
| "rewards/correctness_reward_func": 0.171875, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 130 | |
| }, | |
| { | |
| "completion_length": 28.6484375, | |
| "epoch": 0.1521044992743106, | |
| "grad_norm": 0.42513248914448737, | |
| "kl": 0.35986328125, | |
| "learning_rate": 9.614885510995047e-07, | |
| "loss": 0.0144, | |
| "reward": 0.5390625, | |
| "reward_std": 0.10979307442903519, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 131 | |
| }, | |
| { | |
| "completion_length": 48.9765625, | |
| "epoch": 0.1532656023222061, | |
| "grad_norm": 0.3690713785403208, | |
| "kl": 0.443359375, | |
| "learning_rate": 9.607613001375545e-07, | |
| "loss": 0.0177, | |
| "reward": 0.6015625, | |
| "reward_std": 0.1454593911767006, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 132 | |
| }, | |
| { | |
| "completion_length": 40.62890625, | |
| "epoch": 0.15442670537010159, | |
| "grad_norm": 0.3570306428369935, | |
| "kl": 0.40283203125, | |
| "learning_rate": 9.600275268529806e-07, | |
| "loss": 0.0161, | |
| "reward": 0.595703125, | |
| "reward_std": 0.0766359455883503, | |
| "rewards/correctness_reward_func": 0.1015625, | |
| "rewards/strict_format_reward_func": 0.494140625, | |
| "step": 133 | |
| }, | |
| { | |
| "completion_length": 48.47265625, | |
| "epoch": 0.1555878084179971, | |
| "grad_norm": 0.6699794243427841, | |
| "kl": 0.40283203125, | |
| "learning_rate": 9.592872416327364e-07, | |
| "loss": 0.0161, | |
| "reward": 0.537109375, | |
| "reward_std": 0.1280214972794056, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.482421875, | |
| "step": 134 | |
| }, | |
| { | |
| "completion_length": 43.74609375, | |
| "epoch": 0.1567489114658926, | |
| "grad_norm": 0.8744900024283108, | |
| "kl": 0.42919921875, | |
| "learning_rate": 9.58540454955955e-07, | |
| "loss": 0.0172, | |
| "reward": 0.568359375, | |
| "reward_std": 0.12251138500869274, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.474609375, | |
| "step": 135 | |
| }, | |
| { | |
| "completion_length": 35.98828125, | |
| "epoch": 0.1579100145137881, | |
| "grad_norm": 10.83989251533941, | |
| "kl": 0.43994140625, | |
| "learning_rate": 9.57787177393801e-07, | |
| "loss": 0.0176, | |
| "reward": 0.59765625, | |
| "reward_std": 0.1689911223948002, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 136 | |
| }, | |
| { | |
| "completion_length": 47.265625, | |
| "epoch": 0.1590711175616836, | |
| "grad_norm": 1.3945795045826723, | |
| "kl": 0.4638671875, | |
| "learning_rate": 9.570274196093216e-07, | |
| "loss": 0.0186, | |
| "reward": 0.5, | |
| "reward_std": 0.051385548897087574, | |
| "rewards/correctness_reward_func": 0.015625, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 137 | |
| }, | |
| { | |
| "completion_length": 37.30859375, | |
| "epoch": 0.1602322206095791, | |
| "grad_norm": 1.2085779669420207, | |
| "kl": 0.4755859375, | |
| "learning_rate": 9.562611923572943e-07, | |
| "loss": 0.019, | |
| "reward": 0.587890625, | |
| "reward_std": 0.16475876048207283, | |
| "rewards/correctness_reward_func": 0.109375, | |
| "rewards/strict_format_reward_func": 0.478515625, | |
| "step": 138 | |
| }, | |
| { | |
| "completion_length": 62.33203125, | |
| "epoch": 0.1613933236574746, | |
| "grad_norm": 0.6440211313776459, | |
| "kl": 0.419921875, | |
| "learning_rate": 9.554885064840755e-07, | |
| "loss": 0.0168, | |
| "reward": 0.51171875, | |
| "reward_std": 0.129510547965765, | |
| "rewards/correctness_reward_func": 0.0390625, | |
| "rewards/strict_format_reward_func": 0.47265625, | |
| "step": 139 | |
| }, | |
| { | |
| "completion_length": 42.2734375, | |
| "epoch": 0.1625544267053701, | |
| "grad_norm": 4.184412362449491, | |
| "kl": 0.458984375, | |
| "learning_rate": 9.547093729274473e-07, | |
| "loss": 0.0183, | |
| "reward": 0.56640625, | |
| "reward_std": 0.1319277435541153, | |
| "rewards/correctness_reward_func": 0.0859375, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 140 | |
| }, | |
| { | |
| "completion_length": 30.3203125, | |
| "epoch": 0.16371552975326562, | |
| "grad_norm": 0.9148362673963394, | |
| "kl": 0.4169921875, | |
| "learning_rate": 9.539238027164618e-07, | |
| "loss": 0.0167, | |
| "reward": 0.513671875, | |
| "reward_std": 0.053198445588350296, | |
| "rewards/correctness_reward_func": 0.0234375, | |
| "rewards/strict_format_reward_func": 0.490234375, | |
| "step": 141 | |
| }, | |
| { | |
| "completion_length": 26.02734375, | |
| "epoch": 0.1648766328011611, | |
| "grad_norm": 23.339183218639594, | |
| "kl": 0.4833984375, | |
| "learning_rate": 9.531318069712853e-07, | |
| "loss": 0.0193, | |
| "reward": 0.599609375, | |
| "reward_std": 0.15335902571678162, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.482421875, | |
| "step": 142 | |
| }, | |
| { | |
| "completion_length": 39.70703125, | |
| "epoch": 0.1660377358490566, | |
| "grad_norm": 4.918732336995348, | |
| "kl": 0.546875, | |
| "learning_rate": 9.523333969030412e-07, | |
| "loss": 0.0219, | |
| "reward": 0.623046875, | |
| "reward_std": 0.18786722049117088, | |
| "rewards/correctness_reward_func": 0.1328125, | |
| "rewards/strict_format_reward_func": 0.490234375, | |
| "step": 143 | |
| }, | |
| { | |
| "completion_length": 24.69140625, | |
| "epoch": 0.1671988388969521, | |
| "grad_norm": 2.3195586374061645, | |
| "kl": 0.46875, | |
| "learning_rate": 9.51528583813651e-07, | |
| "loss": 0.0187, | |
| "reward": 0.58203125, | |
| "reward_std": 0.13684775680303574, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.48828125, | |
| "step": 144 | |
| }, | |
| { | |
| "completion_length": 40.7109375, | |
| "epoch": 0.1683599419448476, | |
| "grad_norm": 0.6860447379166801, | |
| "kl": 0.49267578125, | |
| "learning_rate": 9.507173790956745e-07, | |
| "loss": 0.0197, | |
| "reward": 0.583984375, | |
| "reward_std": 0.1592714935541153, | |
| "rewards/correctness_reward_func": 0.1015625, | |
| "rewards/strict_format_reward_func": 0.482421875, | |
| "step": 145 | |
| }, | |
| { | |
| "completion_length": 60.76953125, | |
| "epoch": 0.16952104499274312, | |
| "grad_norm": 1.208065145658311, | |
| "kl": 0.55859375, | |
| "learning_rate": 9.498997942321483e-07, | |
| "loss": 0.0223, | |
| "reward": 0.5390625, | |
| "reward_std": 0.1552504375576973, | |
| "rewards/correctness_reward_func": 0.0703125, | |
| "rewards/strict_format_reward_func": 0.46875, | |
| "step": 146 | |
| }, | |
| { | |
| "completion_length": 34.5859375, | |
| "epoch": 0.1706821480406386, | |
| "grad_norm": 2.412402656297087, | |
| "kl": 0.5146484375, | |
| "learning_rate": 9.490758407964233e-07, | |
| "loss": 0.0206, | |
| "reward": 0.65234375, | |
| "reward_std": 0.14944713562726974, | |
| "rewards/correctness_reward_func": 0.171875, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 147 | |
| }, | |
| { | |
| "completion_length": 33.71875, | |
| "epoch": 0.1718432510885341, | |
| "grad_norm": 1.3484678762791356, | |
| "kl": 0.5146484375, | |
| "learning_rate": 9.482455304520012e-07, | |
| "loss": 0.0206, | |
| "reward": 0.50390625, | |
| "reward_std": 0.033667195588350296, | |
| "rewards/correctness_reward_func": 0.015625, | |
| "rewards/strict_format_reward_func": 0.48828125, | |
| "step": 148 | |
| }, | |
| { | |
| "completion_length": 43.68359375, | |
| "epoch": 0.17300435413642962, | |
| "grad_norm": 4.476292060280118, | |
| "kl": 0.5556640625, | |
| "learning_rate": 9.474088749523687e-07, | |
| "loss": 0.0222, | |
| "reward": 0.51953125, | |
| "reward_std": 0.051385547965765, | |
| "rewards/correctness_reward_func": 0.0390625, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 149 | |
| }, | |
| { | |
| "completion_length": 57.0078125, | |
| "epoch": 0.1741654571843251, | |
| "grad_norm": 3.243977251886513, | |
| "kl": 0.470703125, | |
| "learning_rate": 9.465658861408323e-07, | |
| "loss": 0.0188, | |
| "reward": 0.5703125, | |
| "reward_std": 0.1800113394856453, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 150 | |
| }, | |
| { | |
| "completion_length": 48.234375, | |
| "epoch": 0.1753265602322206, | |
| "grad_norm": 3.965397946904913, | |
| "kl": 0.51220703125, | |
| "learning_rate": 9.457165759503491e-07, | |
| "loss": 0.0204, | |
| "reward": 0.576171875, | |
| "reward_std": 0.14397033769637346, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.482421875, | |
| "step": 151 | |
| }, | |
| { | |
| "completion_length": 27.34765625, | |
| "epoch": 0.1764876632801161, | |
| "grad_norm": 5.956710202169321, | |
| "kl": 0.52734375, | |
| "learning_rate": 9.448609564033593e-07, | |
| "loss": 0.0211, | |
| "reward": 0.521484375, | |
| "reward_std": 0.13792734686285257, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.474609375, | |
| "step": 152 | |
| }, | |
| { | |
| "completion_length": 55.0234375, | |
| "epoch": 0.1776487663280116, | |
| "grad_norm": 1.9878758706657031, | |
| "kl": 0.625, | |
| "learning_rate": 9.439990396116148e-07, | |
| "loss": 0.025, | |
| "reward": 0.486328125, | |
| "reward_std": 0.1357191875576973, | |
| "rewards/correctness_reward_func": 0.03125, | |
| "rewards/strict_format_reward_func": 0.455078125, | |
| "step": 153 | |
| }, | |
| { | |
| "completion_length": 47.39453125, | |
| "epoch": 0.17880986937590712, | |
| "grad_norm": 0.3506677329292148, | |
| "kl": 0.568359375, | |
| "learning_rate": 9.431308377760093e-07, | |
| "loss": 0.0228, | |
| "reward": 0.587890625, | |
| "reward_std": 0.2043701931834221, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.470703125, | |
| "step": 154 | |
| }, | |
| { | |
| "completion_length": 34.31640625, | |
| "epoch": 0.1799709724238026, | |
| "grad_norm": 3.844767491669304, | |
| "kl": 0.51953125, | |
| "learning_rate": 9.422563631864037e-07, | |
| "loss": 0.0208, | |
| "reward": 0.513671875, | |
| "reward_std": 0.1123964935541153, | |
| "rewards/correctness_reward_func": 0.0390625, | |
| "rewards/strict_format_reward_func": 0.474609375, | |
| "step": 155 | |
| }, | |
| { | |
| "completion_length": 35.26171875, | |
| "epoch": 0.1811320754716981, | |
| "grad_norm": 10.543061352265745, | |
| "kl": 0.66748046875, | |
| "learning_rate": 9.413756282214537e-07, | |
| "loss": 0.0267, | |
| "reward": 0.560546875, | |
| "reward_std": 0.2216724045574665, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.466796875, | |
| "step": 156 | |
| }, | |
| { | |
| "completion_length": 53.375, | |
| "epoch": 0.18229317851959362, | |
| "grad_norm": 0.6406847195030275, | |
| "kl": 0.51416015625, | |
| "learning_rate": 9.404886453484335e-07, | |
| "loss": 0.0206, | |
| "reward": 0.486328125, | |
| "reward_std": 0.12331762351095676, | |
| "rewards/correctness_reward_func": 0.03125, | |
| "rewards/strict_format_reward_func": 0.455078125, | |
| "step": 157 | |
| }, | |
| { | |
| "completion_length": 33.31640625, | |
| "epoch": 0.1834542815674891, | |
| "grad_norm": 1.254136142861394, | |
| "kl": 0.6044921875, | |
| "learning_rate": 9.395954271230603e-07, | |
| "loss": 0.0242, | |
| "reward": 0.55078125, | |
| "reward_std": 0.1352296955883503, | |
| "rewards/correctness_reward_func": 0.0703125, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 158 | |
| }, | |
| { | |
| "completion_length": 38.45703125, | |
| "epoch": 0.18461538461538463, | |
| "grad_norm": 1.1951784005374884, | |
| "kl": 0.580078125, | |
| "learning_rate": 9.386959861893157e-07, | |
| "loss": 0.0232, | |
| "reward": 0.5, | |
| "reward_std": 0.09967818111181259, | |
| "rewards/correctness_reward_func": 0.03125, | |
| "rewards/strict_format_reward_func": 0.46875, | |
| "step": 159 | |
| }, | |
| { | |
| "completion_length": 49.9921875, | |
| "epoch": 0.18577648766328012, | |
| "grad_norm": 1.0186965999705826, | |
| "kl": 0.611328125, | |
| "learning_rate": 9.377903352792671e-07, | |
| "loss": 0.0244, | |
| "reward": 0.599609375, | |
| "reward_std": 0.19500285387039185, | |
| "rewards/correctness_reward_func": 0.1328125, | |
| "rewards/strict_format_reward_func": 0.466796875, | |
| "step": 160 | |
| }, | |
| { | |
| "completion_length": 25.6953125, | |
| "epoch": 0.1869375907111756, | |
| "grad_norm": 0.9553165570763839, | |
| "kl": 0.6337890625, | |
| "learning_rate": 9.368784872128876e-07, | |
| "loss": 0.0253, | |
| "reward": 0.65625, | |
| "reward_std": 0.23248206824064255, | |
| "rewards/correctness_reward_func": 0.1796875, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 161 | |
| }, | |
| { | |
| "completion_length": 29.96484375, | |
| "epoch": 0.18809869375907112, | |
| "grad_norm": 1.320529298833274, | |
| "kl": 0.677734375, | |
| "learning_rate": 9.359604548978741e-07, | |
| "loss": 0.0271, | |
| "reward": 0.546875, | |
| "reward_std": 0.1555742807686329, | |
| "rewards/correctness_reward_func": 0.0703125, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 162 | |
| }, | |
| { | |
| "completion_length": 37.44921875, | |
| "epoch": 0.1892597968069666, | |
| "grad_norm": 2.2124875651885754, | |
| "kl": 0.6181640625, | |
| "learning_rate": 9.35036251329465e-07, | |
| "loss": 0.0247, | |
| "reward": 0.513671875, | |
| "reward_std": 0.09588673710823059, | |
| "rewards/correctness_reward_func": 0.0390625, | |
| "rewards/strict_format_reward_func": 0.474609375, | |
| "step": 163 | |
| }, | |
| { | |
| "completion_length": 30.9609375, | |
| "epoch": 0.19042089985486213, | |
| "grad_norm": 0.7762326728671275, | |
| "kl": 0.755859375, | |
| "learning_rate": 9.341058895902562e-07, | |
| "loss": 0.0303, | |
| "reward": 0.615234375, | |
| "reward_std": 0.18324187397956848, | |
| "rewards/correctness_reward_func": 0.1328125, | |
| "rewards/strict_format_reward_func": 0.482421875, | |
| "step": 164 | |
| }, | |
| { | |
| "completion_length": 39.90234375, | |
| "epoch": 0.19158200290275762, | |
| "grad_norm": 1.5333968229845967, | |
| "kl": 0.681640625, | |
| "learning_rate": 9.331693828500158e-07, | |
| "loss": 0.0273, | |
| "reward": 0.53125, | |
| "reward_std": 0.11541798710823059, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 165 | |
| }, | |
| { | |
| "completion_length": 39.2265625, | |
| "epoch": 0.1927431059506531, | |
| "grad_norm": 3.341316216605523, | |
| "kl": 0.6484375, | |
| "learning_rate": 9.322267443654972e-07, | |
| "loss": 0.026, | |
| "reward": 0.552734375, | |
| "reward_std": 0.13462539482861757, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.474609375, | |
| "step": 166 | |
| }, | |
| { | |
| "completion_length": 41.33203125, | |
| "epoch": 0.19390420899854863, | |
| "grad_norm": 0.8334577080111663, | |
| "kl": 0.689453125, | |
| "learning_rate": 9.312779874802525e-07, | |
| "loss": 0.0276, | |
| "reward": 0.72265625, | |
| "reward_std": 0.2527032420039177, | |
| "rewards/correctness_reward_func": 0.2578125, | |
| "rewards/strict_format_reward_func": 0.46484375, | |
| "step": 167 | |
| }, | |
| { | |
| "completion_length": 27.4296875, | |
| "epoch": 0.19506531204644412, | |
| "grad_norm": 0.6225355591311358, | |
| "kl": 0.6796875, | |
| "learning_rate": 9.303231256244428e-07, | |
| "loss": 0.0272, | |
| "reward": 0.572265625, | |
| "reward_std": 0.1169070415198803, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.478515625, | |
| "step": 168 | |
| }, | |
| { | |
| "completion_length": 26.65625, | |
| "epoch": 0.19622641509433963, | |
| "grad_norm": 0.31164342343966356, | |
| "kl": 0.701171875, | |
| "learning_rate": 9.293621723146484e-07, | |
| "loss": 0.0281, | |
| "reward": 0.595703125, | |
| "reward_std": 0.1571781374514103, | |
| "rewards/correctness_reward_func": 0.109375, | |
| "rewards/strict_format_reward_func": 0.486328125, | |
| "step": 169 | |
| }, | |
| { | |
| "completion_length": 32.9609375, | |
| "epoch": 0.19738751814223512, | |
| "grad_norm": 0.8220338043159918, | |
| "kl": 0.6875, | |
| "learning_rate": 9.283951411536772e-07, | |
| "loss": 0.0274, | |
| "reward": 0.59375, | |
| "reward_std": 0.15472763776779175, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 170 | |
| }, | |
| { | |
| "completion_length": 29.8984375, | |
| "epoch": 0.19854862119013061, | |
| "grad_norm": 3.2547872245531146, | |
| "kl": 0.7255859375, | |
| "learning_rate": 9.274220458303726e-07, | |
| "loss": 0.029, | |
| "reward": 0.609375, | |
| "reward_std": 0.2199743092060089, | |
| "rewards/correctness_reward_func": 0.1328125, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 171 | |
| }, | |
| { | |
| "completion_length": 22.08203125, | |
| "epoch": 0.19970972423802613, | |
| "grad_norm": 1.0413843485165826, | |
| "kl": 0.7294921875, | |
| "learning_rate": 9.264429001194192e-07, | |
| "loss": 0.0292, | |
| "reward": 0.595703125, | |
| "reward_std": 0.2031683847308159, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.478515625, | |
| "step": 172 | |
| }, | |
| { | |
| "completion_length": 51.3671875, | |
| "epoch": 0.20087082728592162, | |
| "grad_norm": 3.778010417040405, | |
| "kl": 0.6826171875, | |
| "learning_rate": 9.254577178811482e-07, | |
| "loss": 0.0273, | |
| "reward": 0.6015625, | |
| "reward_std": 0.1914496347308159, | |
| "rewards/correctness_reward_func": 0.1328125, | |
| "rewards/strict_format_reward_func": 0.46875, | |
| "step": 173 | |
| }, | |
| { | |
| "completion_length": 44.84765625, | |
| "epoch": 0.20203193033381714, | |
| "grad_norm": 2.305304587495654, | |
| "kl": 0.7666015625, | |
| "learning_rate": 9.24466513061341e-07, | |
| "loss": 0.0307, | |
| "reward": 0.59375, | |
| "reward_std": 0.1900114268064499, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 174 | |
| }, | |
| { | |
| "completion_length": 36.86328125, | |
| "epoch": 0.20319303338171263, | |
| "grad_norm": 0.7728198355409298, | |
| "kl": 0.712890625, | |
| "learning_rate": 9.234692996910323e-07, | |
| "loss": 0.0285, | |
| "reward": 0.62890625, | |
| "reward_std": 0.1866152435541153, | |
| "rewards/correctness_reward_func": 0.1484375, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 175 | |
| }, | |
| { | |
| "completion_length": 29.98828125, | |
| "epoch": 0.20435413642960812, | |
| "grad_norm": 1.289972465399101, | |
| "kl": 0.6865234375, | |
| "learning_rate": 9.224660918863103e-07, | |
| "loss": 0.0275, | |
| "reward": 0.595703125, | |
| "reward_std": 0.13913594186306, | |
| "rewards/correctness_reward_func": 0.109375, | |
| "rewards/strict_format_reward_func": 0.486328125, | |
| "step": 176 | |
| }, | |
| { | |
| "completion_length": 34.23046875, | |
| "epoch": 0.20551523947750364, | |
| "grad_norm": 0.5793735600612212, | |
| "kl": 0.759765625, | |
| "learning_rate": 9.214569038481181e-07, | |
| "loss": 0.0304, | |
| "reward": 0.5625, | |
| "reward_std": 0.1687820851802826, | |
| "rewards/correctness_reward_func": 0.0859375, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 177 | |
| }, | |
| { | |
| "completion_length": 31.23828125, | |
| "epoch": 0.20667634252539913, | |
| "grad_norm": 0.5984526162665614, | |
| "kl": 0.771484375, | |
| "learning_rate": 9.20441749862052e-07, | |
| "loss": 0.0309, | |
| "reward": 0.68359375, | |
| "reward_std": 0.1762864589691162, | |
| "rewards/correctness_reward_func": 0.1953125, | |
| "rewards/strict_format_reward_func": 0.48828125, | |
| "step": 178 | |
| }, | |
| { | |
| "completion_length": 38.296875, | |
| "epoch": 0.20783744557329464, | |
| "grad_norm": 1.181839300217833, | |
| "kl": 0.7099609375, | |
| "learning_rate": 9.1942064429816e-07, | |
| "loss": 0.0284, | |
| "reward": 0.548828125, | |
| "reward_std": 0.10820978879928589, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.470703125, | |
| "step": 179 | |
| }, | |
| { | |
| "completion_length": 35.44921875, | |
| "epoch": 0.20899854862119013, | |
| "grad_norm": 7.06190367002311, | |
| "kl": 0.7001953125, | |
| "learning_rate": 9.18393601610737e-07, | |
| "loss": 0.028, | |
| "reward": 0.56640625, | |
| "reward_std": 0.1620996668934822, | |
| "rewards/correctness_reward_func": 0.0859375, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 180 | |
| }, | |
| { | |
| "completion_length": 24.76953125, | |
| "epoch": 0.21015965166908562, | |
| "grad_norm": 0.4282018680025159, | |
| "kl": 0.7275390625, | |
| "learning_rate": 9.173606363381218e-07, | |
| "loss": 0.0291, | |
| "reward": 0.654296875, | |
| "reward_std": 0.2599649503827095, | |
| "rewards/correctness_reward_func": 0.1640625, | |
| "rewards/strict_format_reward_func": 0.490234375, | |
| "step": 181 | |
| }, | |
| { | |
| "completion_length": 61.73046875, | |
| "epoch": 0.21132075471698114, | |
| "grad_norm": 0.666578405400966, | |
| "kl": 0.705078125, | |
| "learning_rate": 9.1632176310249e-07, | |
| "loss": 0.0283, | |
| "reward": 0.54296875, | |
| "reward_std": 0.1953125, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.46484375, | |
| "step": 182 | |
| }, | |
| { | |
| "completion_length": 41.6875, | |
| "epoch": 0.21248185776487663, | |
| "grad_norm": 2.4086312779043495, | |
| "kl": 0.6748046875, | |
| "learning_rate": 9.152769966096481e-07, | |
| "loss": 0.027, | |
| "reward": 0.65625, | |
| "reward_std": 0.2158661186695099, | |
| "rewards/correctness_reward_func": 0.1796875, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 183 | |
| }, | |
| { | |
| "completion_length": 47.5078125, | |
| "epoch": 0.21364296081277212, | |
| "grad_norm": 0.7460531004888707, | |
| "kl": 0.6865234375, | |
| "learning_rate": 9.142263516488244e-07, | |
| "loss": 0.0275, | |
| "reward": 0.53125, | |
| "reward_std": 0.1276262328028679, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 184 | |
| }, | |
| { | |
| "completion_length": 38.33203125, | |
| "epoch": 0.21480406386066764, | |
| "grad_norm": 1.677852493854183, | |
| "kl": 0.6611328125, | |
| "learning_rate": 9.131698430924605e-07, | |
| "loss": 0.0264, | |
| "reward": 0.6015625, | |
| "reward_std": 0.1689911223948002, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 185 | |
| }, | |
| { | |
| "completion_length": 31.5859375, | |
| "epoch": 0.21596516690856313, | |
| "grad_norm": 21.23887712764663, | |
| "kl": 0.689453125, | |
| "learning_rate": 9.121074858959996e-07, | |
| "loss": 0.0275, | |
| "reward": 0.564453125, | |
| "reward_std": 0.09907388500869274, | |
| "rewards/correctness_reward_func": 0.0859375, | |
| "rewards/strict_format_reward_func": 0.478515625, | |
| "step": 186 | |
| }, | |
| { | |
| "completion_length": 41.9453125, | |
| "epoch": 0.21712626995645865, | |
| "grad_norm": 0.6508903252265714, | |
| "kl": 0.6748046875, | |
| "learning_rate": 9.110392950976764e-07, | |
| "loss": 0.027, | |
| "reward": 0.5234375, | |
| "reward_std": 0.137323047965765, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.46875, | |
| "step": 187 | |
| }, | |
| { | |
| "completion_length": 62.2109375, | |
| "epoch": 0.21828737300435413, | |
| "grad_norm": 1.1278006956369315, | |
| "kl": 0.673828125, | |
| "learning_rate": 9.099652858183027e-07, | |
| "loss": 0.027, | |
| "reward": 0.56640625, | |
| "reward_std": 0.1318129375576973, | |
| "rewards/correctness_reward_func": 0.1015625, | |
| "rewards/strict_format_reward_func": 0.46484375, | |
| "step": 188 | |
| }, | |
| { | |
| "completion_length": 46.51171875, | |
| "epoch": 0.21944847605224962, | |
| "grad_norm": 1.1019281044279183, | |
| "kl": 0.7548828125, | |
| "learning_rate": 9.088854732610542e-07, | |
| "loss": 0.0302, | |
| "reward": 0.572265625, | |
| "reward_std": 0.1169070415198803, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.478515625, | |
| "step": 189 | |
| }, | |
| { | |
| "completion_length": 35.03125, | |
| "epoch": 0.22060957910014514, | |
| "grad_norm": 14.962735092929739, | |
| "kl": 0.6845703125, | |
| "learning_rate": 9.077998727112552e-07, | |
| "loss": 0.0274, | |
| "reward": 0.564453125, | |
| "reward_std": 0.1369277872145176, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.470703125, | |
| "step": 190 | |
| }, | |
| { | |
| "completion_length": 55.27734375, | |
| "epoch": 0.22177068214804063, | |
| "grad_norm": 0.3504770063047999, | |
| "kl": 0.654296875, | |
| "learning_rate": 9.067084995361623e-07, | |
| "loss": 0.0262, | |
| "reward": 0.55078125, | |
| "reward_std": 0.1510637328028679, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.47265625, | |
| "step": 191 | |
| }, | |
| { | |
| "completion_length": 47.34375, | |
| "epoch": 0.22293178519593615, | |
| "grad_norm": 1.8159467800873224, | |
| "kl": 0.7333984375, | |
| "learning_rate": 9.05611369184746e-07, | |
| "loss": 0.0293, | |
| "reward": 0.57421875, | |
| "reward_std": 0.1376468911767006, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 192 | |
| }, | |
| { | |
| "completion_length": 49.0390625, | |
| "epoch": 0.22409288824383164, | |
| "grad_norm": 3.120448246962212, | |
| "kl": 0.69140625, | |
| "learning_rate": 9.045084971874737e-07, | |
| "loss": 0.0277, | |
| "reward": 0.609375, | |
| "reward_std": 0.1610843911767006, | |
| "rewards/correctness_reward_func": 0.1328125, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 193 | |
| }, | |
| { | |
| "completion_length": 33.7421875, | |
| "epoch": 0.22525399129172713, | |
| "grad_norm": 2.5542170019130244, | |
| "kl": 0.6787109375, | |
| "learning_rate": 9.03399899156088e-07, | |
| "loss": 0.0272, | |
| "reward": 0.58984375, | |
| "reward_std": 0.1015625, | |
| "rewards/correctness_reward_func": 0.1015625, | |
| "rewards/strict_format_reward_func": 0.48828125, | |
| "step": 194 | |
| }, | |
| { | |
| "completion_length": 54.81640625, | |
| "epoch": 0.22641509433962265, | |
| "grad_norm": 0.6406888848138048, | |
| "kl": 0.7216796875, | |
| "learning_rate": 9.022855907833871e-07, | |
| "loss": 0.0288, | |
| "reward": 0.5390625, | |
| "reward_std": 0.1084902435541153, | |
| "rewards/correctness_reward_func": 0.0703125, | |
| "rewards/strict_format_reward_func": 0.46875, | |
| "step": 195 | |
| }, | |
| { | |
| "completion_length": 33.41796875, | |
| "epoch": 0.22757619738751814, | |
| "grad_norm": 0.3997318900990298, | |
| "kl": 0.7177734375, | |
| "learning_rate": 9.011655878430018e-07, | |
| "loss": 0.0287, | |
| "reward": 0.587890625, | |
| "reward_std": 0.16475242003798485, | |
| "rewards/correctness_reward_func": 0.109375, | |
| "rewards/strict_format_reward_func": 0.478515625, | |
| "step": 196 | |
| }, | |
| { | |
| "completion_length": 62.3671875, | |
| "epoch": 0.22873730043541365, | |
| "grad_norm": 1.585750352753696, | |
| "kl": 0.71484375, | |
| "learning_rate": 9.000399061891728e-07, | |
| "loss": 0.0286, | |
| "reward": 0.630859375, | |
| "reward_std": 0.141229297965765, | |
| "rewards/correctness_reward_func": 0.1484375, | |
| "rewards/strict_format_reward_func": 0.482421875, | |
| "step": 197 | |
| }, | |
| { | |
| "completion_length": 38.67578125, | |
| "epoch": 0.22989840348330914, | |
| "grad_norm": 5.358536856283286, | |
| "kl": 0.6640625, | |
| "learning_rate": 8.98908561756526e-07, | |
| "loss": 0.0265, | |
| "reward": 0.517578125, | |
| "reward_std": 0.055291798897087574, | |
| "rewards/correctness_reward_func": 0.0390625, | |
| "rewards/strict_format_reward_func": 0.478515625, | |
| "step": 198 | |
| }, | |
| { | |
| "completion_length": 30.2109375, | |
| "epoch": 0.23105950653120463, | |
| "grad_norm": 0.39601285118928503, | |
| "kl": 0.720703125, | |
| "learning_rate": 8.977715705598467e-07, | |
| "loss": 0.0288, | |
| "reward": 0.54296875, | |
| "reward_std": 0.09165100753307343, | |
| "rewards/correctness_reward_func": 0.0625, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 199 | |
| }, | |
| { | |
| "completion_length": 29.21484375, | |
| "epoch": 0.23222060957910015, | |
| "grad_norm": 1.491626514094341, | |
| "kl": 0.73046875, | |
| "learning_rate": 8.966289486938536e-07, | |
| "loss": 0.0292, | |
| "reward": 0.63671875, | |
| "reward_std": 0.1542871668934822, | |
| "rewards/correctness_reward_func": 0.15625, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 200 | |
| }, | |
| { | |
| "completion_length": 46.3046875, | |
| "epoch": 0.23338171262699564, | |
| "grad_norm": 0.6565437407352915, | |
| "kl": 0.6962890625, | |
| "learning_rate": 8.954807123329703e-07, | |
| "loss": 0.0278, | |
| "reward": 0.51953125, | |
| "reward_std": 0.1006777435541153, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.47265625, | |
| "step": 201 | |
| }, | |
| { | |
| "completion_length": 30.859375, | |
| "epoch": 0.23454281567489116, | |
| "grad_norm": 0.32433472081782594, | |
| "kl": 0.6640625, | |
| "learning_rate": 8.943268777310964e-07, | |
| "loss": 0.0266, | |
| "reward": 0.515625, | |
| "reward_std": 0.09375, | |
| "rewards/correctness_reward_func": 0.03125, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 202 | |
| }, | |
| { | |
| "completion_length": 38.45703125, | |
| "epoch": 0.23570391872278665, | |
| "grad_norm": 0.41559508425580927, | |
| "kl": 0.6484375, | |
| "learning_rate": 8.931674612213778e-07, | |
| "loss": 0.0259, | |
| "reward": 0.55078125, | |
| "reward_std": 0.082635547965765, | |
| "rewards/correctness_reward_func": 0.0703125, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 203 | |
| }, | |
| { | |
| "completion_length": 43.28515625, | |
| "epoch": 0.23686502177068214, | |
| "grad_norm": 0.8867725325822029, | |
| "kl": 0.6962890625, | |
| "learning_rate": 8.920024792159754e-07, | |
| "loss": 0.0279, | |
| "reward": 0.51953125, | |
| "reward_std": 0.11871993914246559, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.47265625, | |
| "step": 204 | |
| }, | |
| { | |
| "completion_length": 23.59375, | |
| "epoch": 0.23802612481857766, | |
| "grad_norm": 0.510002854031267, | |
| "kl": 0.748046875, | |
| "learning_rate": 8.908319482058324e-07, | |
| "loss": 0.0299, | |
| "reward": 0.578125, | |
| "reward_std": 0.10628208518028259, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 205 | |
| }, | |
| { | |
| "completion_length": 32.55078125, | |
| "epoch": 0.23918722786647315, | |
| "grad_norm": 0.35776972668521406, | |
| "kl": 0.775390625, | |
| "learning_rate": 8.896558847604413e-07, | |
| "loss": 0.031, | |
| "reward": 0.6875, | |
| "reward_std": 0.2245633751153946, | |
| "rewards/correctness_reward_func": 0.1953125, | |
| "rewards/strict_format_reward_func": 0.4921875, | |
| "step": 206 | |
| }, | |
| { | |
| "completion_length": 29.95703125, | |
| "epoch": 0.24034833091436866, | |
| "grad_norm": 1.1414350140139438, | |
| "kl": 0.755859375, | |
| "learning_rate": 8.884743055276091e-07, | |
| "loss": 0.0302, | |
| "reward": 0.6171875, | |
| "reward_std": 0.1635015830397606, | |
| "rewards/correctness_reward_func": 0.1328125, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 207 | |
| }, | |
| { | |
| "completion_length": 34.19921875, | |
| "epoch": 0.24150943396226415, | |
| "grad_norm": 0.44449352427969546, | |
| "kl": 0.7802734375, | |
| "learning_rate": 8.872872272332219e-07, | |
| "loss": 0.0312, | |
| "reward": 0.6171875, | |
| "reward_std": 0.1620996706187725, | |
| "rewards/correctness_reward_func": 0.1328125, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 208 | |
| }, | |
| { | |
| "completion_length": 57.421875, | |
| "epoch": 0.24267053701015964, | |
| "grad_norm": 0.8888526828073323, | |
| "kl": 0.693359375, | |
| "learning_rate": 8.860946666810077e-07, | |
| "loss": 0.0277, | |
| "reward": 0.546875, | |
| "reward_std": 0.115094143897295, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.46875, | |
| "step": 209 | |
| }, | |
| { | |
| "completion_length": 49.26953125, | |
| "epoch": 0.24383164005805516, | |
| "grad_norm": 0.4662997492945863, | |
| "kl": 0.6845703125, | |
| "learning_rate": 8.848966407522991e-07, | |
| "loss": 0.0274, | |
| "reward": 0.515625, | |
| "reward_std": 0.0893336832523346, | |
| "rewards/correctness_reward_func": 0.0390625, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 210 | |
| }, | |
| { | |
| "completion_length": 46.41796875, | |
| "epoch": 0.24499274310595065, | |
| "grad_norm": 0.330950605212178, | |
| "kl": 0.701171875, | |
| "learning_rate": 8.836931664057934e-07, | |
| "loss": 0.0281, | |
| "reward": 0.576171875, | |
| "reward_std": 0.1290367692708969, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.482421875, | |
| "step": 211 | |
| }, | |
| { | |
| "completion_length": 48.34765625, | |
| "epoch": 0.24615384615384617, | |
| "grad_norm": 0.4398380701808975, | |
| "kl": 0.6806640625, | |
| "learning_rate": 8.824842606773141e-07, | |
| "loss": 0.0272, | |
| "reward": 0.546875, | |
| "reward_std": 0.1642715372145176, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.46875, | |
| "step": 212 | |
| }, | |
| { | |
| "completion_length": 34.70703125, | |
| "epoch": 0.24731494920174166, | |
| "grad_norm": 0.5212809108263435, | |
| "kl": 0.7197265625, | |
| "learning_rate": 8.81269940679568e-07, | |
| "loss": 0.0288, | |
| "reward": 0.572265625, | |
| "reward_std": 0.0900527872145176, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.478515625, | |
| "step": 213 | |
| }, | |
| { | |
| "completion_length": 36.984375, | |
| "epoch": 0.24847605224963715, | |
| "grad_norm": 0.4642489160745716, | |
| "kl": 0.7744140625, | |
| "learning_rate": 8.800502236019043e-07, | |
| "loss": 0.031, | |
| "reward": 0.634765625, | |
| "reward_std": 0.17212741822004318, | |
| "rewards/correctness_reward_func": 0.15625, | |
| "rewards/strict_format_reward_func": 0.478515625, | |
| "step": 214 | |
| }, | |
| { | |
| "completion_length": 43.8046875, | |
| "epoch": 0.24963715529753266, | |
| "grad_norm": 0.548886691397983, | |
| "kl": 0.640625, | |
| "learning_rate": 8.788251267100704e-07, | |
| "loss": 0.0256, | |
| "reward": 0.537109375, | |
| "reward_std": 0.14466177113354206, | |
| "rewards/correctness_reward_func": 0.0625, | |
| "rewards/strict_format_reward_func": 0.474609375, | |
| "step": 215 | |
| }, | |
| { | |
| "completion_length": 30.15234375, | |
| "epoch": 0.2507982583454282, | |
| "grad_norm": 0.2595648862557409, | |
| "kl": 0.7314453125, | |
| "learning_rate": 8.77594667345968e-07, | |
| "loss": 0.0293, | |
| "reward": 0.59765625, | |
| "reward_std": 0.1171875, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 216 | |
| }, | |
| { | |
| "completion_length": 54.96875, | |
| "epoch": 0.25195936139332364, | |
| "grad_norm": 2.2269504312204096, | |
| "kl": 0.6982421875, | |
| "learning_rate": 8.763588629274076e-07, | |
| "loss": 0.0279, | |
| "reward": 0.541015625, | |
| "reward_std": 0.125604297965765, | |
| "rewards/correctness_reward_func": 0.0703125, | |
| "rewards/strict_format_reward_func": 0.470703125, | |
| "step": 217 | |
| }, | |
| { | |
| "completion_length": 45.328125, | |
| "epoch": 0.25312046444121916, | |
| "grad_norm": 0.622963472385017, | |
| "kl": 0.65625, | |
| "learning_rate": 8.751177309478617e-07, | |
| "loss": 0.0262, | |
| "reward": 0.49609375, | |
| "reward_std": 0.12400043848901987, | |
| "rewards/correctness_reward_func": 0.03125, | |
| "rewards/strict_format_reward_func": 0.46484375, | |
| "step": 218 | |
| }, | |
| { | |
| "completion_length": 48.9140625, | |
| "epoch": 0.2542815674891147, | |
| "grad_norm": 0.3953561223214397, | |
| "kl": 0.740234375, | |
| "learning_rate": 8.73871288976217e-07, | |
| "loss": 0.0296, | |
| "reward": 0.5859375, | |
| "reward_std": 0.1987086832523346, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.46875, | |
| "step": 219 | |
| }, | |
| { | |
| "completion_length": 42.93359375, | |
| "epoch": 0.25544267053701014, | |
| "grad_norm": 0.346216440100207, | |
| "kl": 0.7021484375, | |
| "learning_rate": 8.726195546565262e-07, | |
| "loss": 0.0281, | |
| "reward": 0.5390625, | |
| "reward_std": 0.140625, | |
| "rewards/correctness_reward_func": 0.0625, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 220 | |
| }, | |
| { | |
| "completion_length": 36.3046875, | |
| "epoch": 0.25660377358490566, | |
| "grad_norm": 1.5813028590873024, | |
| "kl": 0.7578125, | |
| "learning_rate": 8.713625457077585e-07, | |
| "loss": 0.0303, | |
| "reward": 0.568359375, | |
| "reward_std": 0.12771336920559406, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.490234375, | |
| "step": 221 | |
| }, | |
| { | |
| "completion_length": 44.7265625, | |
| "epoch": 0.2577648766328012, | |
| "grad_norm": 1.9405311075755103, | |
| "kl": 0.8466796875, | |
| "learning_rate": 8.701002799235474e-07, | |
| "loss": 0.0339, | |
| "reward": 0.59375, | |
| "reward_std": 0.17461559176445007, | |
| "rewards/correctness_reward_func": 0.125, | |
| "rewards/strict_format_reward_func": 0.46875, | |
| "step": 222 | |
| }, | |
| { | |
| "completion_length": 44.08203125, | |
| "epoch": 0.25892597968069664, | |
| "grad_norm": 0.7411664029084547, | |
| "kl": 0.7255859375, | |
| "learning_rate": 8.688327751719402e-07, | |
| "loss": 0.029, | |
| "reward": 0.552734375, | |
| "reward_std": 0.1280214935541153, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.474609375, | |
| "step": 223 | |
| }, | |
| { | |
| "completion_length": 52.69921875, | |
| "epoch": 0.26008708272859216, | |
| "grad_norm": 0.46133780141282477, | |
| "kl": 0.7353515625, | |
| "learning_rate": 8.675600493951447e-07, | |
| "loss": 0.0294, | |
| "reward": 0.609375, | |
| "reward_std": 0.1139797791838646, | |
| "rewards/correctness_reward_func": 0.140625, | |
| "rewards/strict_format_reward_func": 0.46875, | |
| "step": 224 | |
| }, | |
| { | |
| "completion_length": 38.2578125, | |
| "epoch": 0.2612481857764877, | |
| "grad_norm": 1.0318467725501794, | |
| "kl": 0.6982421875, | |
| "learning_rate": 8.662821206092747e-07, | |
| "loss": 0.0279, | |
| "reward": 0.587890625, | |
| "reward_std": 0.2320011854171753, | |
| "rewards/correctness_reward_func": 0.109375, | |
| "rewards/strict_format_reward_func": 0.478515625, | |
| "step": 225 | |
| }, | |
| { | |
| "completion_length": 45.69140625, | |
| "epoch": 0.2624092888243832, | |
| "grad_norm": 0.22277480044336392, | |
| "kl": 0.75, | |
| "learning_rate": 8.64999006904096e-07, | |
| "loss": 0.03, | |
| "reward": 0.513671875, | |
| "reward_std": 0.094354297965765, | |
| "rewards/correctness_reward_func": 0.0390625, | |
| "rewards/strict_format_reward_func": 0.474609375, | |
| "step": 226 | |
| }, | |
| { | |
| "completion_length": 32.74609375, | |
| "epoch": 0.26357039187227865, | |
| "grad_norm": 0.4780303420088376, | |
| "kl": 0.7216796875, | |
| "learning_rate": 8.637107264427688e-07, | |
| "loss": 0.0289, | |
| "reward": 0.587890625, | |
| "reward_std": 0.2041836604475975, | |
| "rewards/correctness_reward_func": 0.109375, | |
| "rewards/strict_format_reward_func": 0.478515625, | |
| "step": 227 | |
| }, | |
| { | |
| "completion_length": 29.2421875, | |
| "epoch": 0.26473149492017417, | |
| "grad_norm": 0.38056082312023104, | |
| "kl": 0.8544921875, | |
| "learning_rate": 8.624172974615926e-07, | |
| "loss": 0.0342, | |
| "reward": 0.5859375, | |
| "reward_std": 0.2013554833829403, | |
| "rewards/correctness_reward_func": 0.109375, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 228 | |
| }, | |
| { | |
| "completion_length": 44.23828125, | |
| "epoch": 0.2658925979680697, | |
| "grad_norm": 0.39159055696334416, | |
| "kl": 0.794921875, | |
| "learning_rate": 8.611187382697458e-07, | |
| "loss": 0.0317, | |
| "reward": 0.548828125, | |
| "reward_std": 0.08323984593153, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.470703125, | |
| "step": 229 | |
| }, | |
| { | |
| "completion_length": 44.40625, | |
| "epoch": 0.26705370101596515, | |
| "grad_norm": 0.34689298597919604, | |
| "kl": 0.7890625, | |
| "learning_rate": 8.598150672490288e-07, | |
| "loss": 0.0316, | |
| "reward": 0.59765625, | |
| "reward_std": 0.1017715372145176, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 230 | |
| }, | |
| { | |
| "completion_length": 35.9921875, | |
| "epoch": 0.26821480406386067, | |
| "grad_norm": 0.45517531147810925, | |
| "kl": 0.7734375, | |
| "learning_rate": 8.585063028536015e-07, | |
| "loss": 0.0309, | |
| "reward": 0.564453125, | |
| "reward_std": 0.1468336395919323, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.470703125, | |
| "step": 231 | |
| }, | |
| { | |
| "completion_length": 36.65625, | |
| "epoch": 0.2693759071117562, | |
| "grad_norm": 2.8712605854584963, | |
| "kl": 0.765625, | |
| "learning_rate": 8.571924636097243e-07, | |
| "loss": 0.0307, | |
| "reward": 0.56640625, | |
| "reward_std": 0.1352296955883503, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.48828125, | |
| "step": 232 | |
| }, | |
| { | |
| "completion_length": 25.33203125, | |
| "epoch": 0.27053701015965165, | |
| "grad_norm": 0.24431120059016243, | |
| "kl": 0.79296875, | |
| "learning_rate": 8.558735681154943e-07, | |
| "loss": 0.0317, | |
| "reward": 0.578125, | |
| "reward_std": 0.1430421955883503, | |
| "rewards/correctness_reward_func": 0.0859375, | |
| "rewards/strict_format_reward_func": 0.4921875, | |
| "step": 233 | |
| }, | |
| { | |
| "completion_length": 25.3203125, | |
| "epoch": 0.27169811320754716, | |
| "grad_norm": 1.2698794348386706, | |
| "kl": 0.802734375, | |
| "learning_rate": 8.545496350405824e-07, | |
| "loss": 0.0321, | |
| "reward": 0.55078125, | |
| "reward_std": 0.11388554889708757, | |
| "rewards/correctness_reward_func": 0.0703125, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 234 | |
| }, | |
| { | |
| "completion_length": 21.9140625, | |
| "epoch": 0.2728592162554427, | |
| "grad_norm": 0.2651240813655857, | |
| "kl": 0.798828125, | |
| "learning_rate": 8.532206831259695e-07, | |
| "loss": 0.032, | |
| "reward": 0.546875, | |
| "reward_std": 0.11332463473081589, | |
| "rewards/correctness_reward_func": 0.0625, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 235 | |
| }, | |
| { | |
| "completion_length": 25.6015625, | |
| "epoch": 0.27402031930333814, | |
| "grad_norm": 0.6783750195102723, | |
| "kl": 0.7880859375, | |
| "learning_rate": 8.518867311836807e-07, | |
| "loss": 0.0315, | |
| "reward": 0.673828125, | |
| "reward_std": 0.23925277311354876, | |
| "rewards/correctness_reward_func": 0.1875, | |
| "rewards/strict_format_reward_func": 0.486328125, | |
| "step": 236 | |
| }, | |
| { | |
| "completion_length": 47.40625, | |
| "epoch": 0.27518142235123366, | |
| "grad_norm": 0.4433767128864105, | |
| "kl": 0.814453125, | |
| "learning_rate": 8.50547798096519e-07, | |
| "loss": 0.0326, | |
| "reward": 0.5234375, | |
| "reward_std": 0.11280747503042221, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 237 | |
| }, | |
| { | |
| "completion_length": 27.6796875, | |
| "epoch": 0.2763425253991292, | |
| "grad_norm": 0.39357743446474786, | |
| "kl": 0.7763671875, | |
| "learning_rate": 8.492039028177984e-07, | |
| "loss": 0.031, | |
| "reward": 0.591796875, | |
| "reward_std": 0.1605743244290352, | |
| "rewards/correctness_reward_func": 0.109375, | |
| "rewards/strict_format_reward_func": 0.482421875, | |
| "step": 238 | |
| }, | |
| { | |
| "completion_length": 23.9453125, | |
| "epoch": 0.2775036284470247, | |
| "grad_norm": 0.32648390222909296, | |
| "kl": 0.767578125, | |
| "learning_rate": 8.478550643710753e-07, | |
| "loss": 0.0307, | |
| "reward": 0.60546875, | |
| "reward_std": 0.1869390830397606, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.48828125, | |
| "step": 239 | |
| }, | |
| { | |
| "completion_length": 37.671875, | |
| "epoch": 0.27866473149492016, | |
| "grad_norm": 0.35588615244881977, | |
| "kl": 0.8017578125, | |
| "learning_rate": 8.465013018498794e-07, | |
| "loss": 0.0321, | |
| "reward": 0.560546875, | |
| "reward_std": 0.16015625, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.482421875, | |
| "step": 240 | |
| }, | |
| { | |
| "completion_length": 31.61328125, | |
| "epoch": 0.2798258345428157, | |
| "grad_norm": 0.4154855280252823, | |
| "kl": 0.8212890625, | |
| "learning_rate": 8.451426344174432e-07, | |
| "loss": 0.0328, | |
| "reward": 0.57421875, | |
| "reward_std": 0.17284834384918213, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 241 | |
| }, | |
| { | |
| "completion_length": 21.45703125, | |
| "epoch": 0.2809869375907112, | |
| "grad_norm": 0.2962320153426291, | |
| "kl": 0.85546875, | |
| "learning_rate": 8.437790813064305e-07, | |
| "loss": 0.0342, | |
| "reward": 0.65234375, | |
| "reward_std": 0.1420426331460476, | |
| "rewards/correctness_reward_func": 0.171875, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 242 | |
| }, | |
| { | |
| "completion_length": 29.56640625, | |
| "epoch": 0.28214804063860666, | |
| "grad_norm": 0.5104498326555604, | |
| "kl": 0.75, | |
| "learning_rate": 8.424106618186653e-07, | |
| "loss": 0.03, | |
| "reward": 0.59765625, | |
| "reward_std": 0.1532718911767006, | |
| "rewards/correctness_reward_func": 0.109375, | |
| "rewards/strict_format_reward_func": 0.48828125, | |
| "step": 243 | |
| }, | |
| { | |
| "completion_length": 28.71875, | |
| "epoch": 0.2833091436865022, | |
| "grad_norm": 1.3952647871604391, | |
| "kl": 0.7685546875, | |
| "learning_rate": 8.410373953248568e-07, | |
| "loss": 0.0308, | |
| "reward": 0.5390625, | |
| "reward_std": 0.10100158676505089, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 244 | |
| }, | |
| { | |
| "completion_length": 25.4765625, | |
| "epoch": 0.2844702467343977, | |
| "grad_norm": 0.3488892885570999, | |
| "kl": 0.759765625, | |
| "learning_rate": 8.396593012643271e-07, | |
| "loss": 0.0304, | |
| "reward": 0.55078125, | |
| "reward_std": 0.1039796955883503, | |
| "rewards/correctness_reward_func": 0.0625, | |
| "rewards/strict_format_reward_func": 0.48828125, | |
| "step": 245 | |
| }, | |
| { | |
| "completion_length": 36.640625, | |
| "epoch": 0.28563134978229315, | |
| "grad_norm": 0.8252384593648274, | |
| "kl": 0.7177734375, | |
| "learning_rate": 8.382763991447343e-07, | |
| "loss": 0.0287, | |
| "reward": 0.580078125, | |
| "reward_std": 0.1716102585196495, | |
| "rewards/correctness_reward_func": 0.1015625, | |
| "rewards/strict_format_reward_func": 0.478515625, | |
| "step": 246 | |
| }, | |
| { | |
| "completion_length": 39.62109375, | |
| "epoch": 0.28679245283018867, | |
| "grad_norm": 9.474712584278485, | |
| "kl": 0.697265625, | |
| "learning_rate": 8.368887085417977e-07, | |
| "loss": 0.0279, | |
| "reward": 0.560546875, | |
| "reward_std": 0.1315324828028679, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.482421875, | |
| "step": 247 | |
| }, | |
| { | |
| "completion_length": 32.84375, | |
| "epoch": 0.2879535558780842, | |
| "grad_norm": 0.5120202650685787, | |
| "kl": 0.7353515625, | |
| "learning_rate": 8.354962490990201e-07, | |
| "loss": 0.0294, | |
| "reward": 0.513671875, | |
| "reward_std": 0.0844484455883503, | |
| "rewards/correctness_reward_func": 0.03125, | |
| "rewards/strict_format_reward_func": 0.482421875, | |
| "step": 248 | |
| }, | |
| { | |
| "completion_length": 20.99609375, | |
| "epoch": 0.2891146589259797, | |
| "grad_norm": 1.0700667980733463, | |
| "kl": 0.6875, | |
| "learning_rate": 8.340990405274091e-07, | |
| "loss": 0.0275, | |
| "reward": 0.53125, | |
| "reward_std": 0.0805421955883503, | |
| "rewards/correctness_reward_func": 0.0390625, | |
| "rewards/strict_format_reward_func": 0.4921875, | |
| "step": 249 | |
| }, | |
| { | |
| "completion_length": 35.40234375, | |
| "epoch": 0.29027576197387517, | |
| "grad_norm": 1.0418024319085948, | |
| "kl": 0.91015625, | |
| "learning_rate": 8.326971026051999e-07, | |
| "loss": 0.0364, | |
| "reward": 0.55859375, | |
| "reward_std": 0.09979298897087574, | |
| "rewards/correctness_reward_func": 0.0703125, | |
| "rewards/strict_format_reward_func": 0.48828125, | |
| "step": 250 | |
| }, | |
| { | |
| "completion_length": 23.42578125, | |
| "epoch": 0.2914368650217707, | |
| "grad_norm": 0.9720263563682217, | |
| "kl": 0.873046875, | |
| "learning_rate": 8.31290455177573e-07, | |
| "loss": 0.0349, | |
| "reward": 0.576171875, | |
| "reward_std": 0.14594027400016785, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.482421875, | |
| "step": 251 | |
| }, | |
| { | |
| "completion_length": 30.20703125, | |
| "epoch": 0.2925979680696662, | |
| "grad_norm": 0.773176913897642, | |
| "kl": 0.6806640625, | |
| "learning_rate": 8.298791181563754e-07, | |
| "loss": 0.0273, | |
| "reward": 0.583984375, | |
| "reward_std": 0.11811564117670059, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.490234375, | |
| "step": 252 | |
| }, | |
| { | |
| "completion_length": 46.68359375, | |
| "epoch": 0.29375907111756167, | |
| "grad_norm": 1.003168597159282, | |
| "kl": 0.6484375, | |
| "learning_rate": 8.28463111519837e-07, | |
| "loss": 0.0259, | |
| "reward": 0.5234375, | |
| "reward_std": 0.1175113394856453, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 253 | |
| }, | |
| { | |
| "completion_length": 28.07421875, | |
| "epoch": 0.2949201741654572, | |
| "grad_norm": 0.4721784825997983, | |
| "kl": 0.6435546875, | |
| "learning_rate": 8.270424553122898e-07, | |
| "loss": 0.0257, | |
| "reward": 0.544921875, | |
| "reward_std": 0.0844484455883503, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.490234375, | |
| "step": 254 | |
| }, | |
| { | |
| "completion_length": 29.39453125, | |
| "epoch": 0.2960812772133527, | |
| "grad_norm": 0.8645019103999428, | |
| "kl": 0.654296875, | |
| "learning_rate": 8.256171696438816e-07, | |
| "loss": 0.0261, | |
| "reward": 0.578125, | |
| "reward_std": 0.1610843874514103, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 255 | |
| }, | |
| { | |
| "completion_length": 31.328125, | |
| "epoch": 0.29724238026124816, | |
| "grad_norm": 1.0576590786193392, | |
| "kl": 0.6591796875, | |
| "learning_rate": 8.241872746902934e-07, | |
| "loss": 0.0263, | |
| "reward": 0.5234375, | |
| "reward_std": 0.09175087884068489, | |
| "rewards/correctness_reward_func": 0.0390625, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 256 | |
| }, | |
| { | |
| "completion_length": 49.671875, | |
| "epoch": 0.2984034833091437, | |
| "grad_norm": 2.5033828252696915, | |
| "kl": 0.6357421875, | |
| "learning_rate": 8.22752790692453e-07, | |
| "loss": 0.0254, | |
| "reward": 0.56640625, | |
| "reward_std": 0.1518699713051319, | |
| "rewards/correctness_reward_func": 0.0859375, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 257 | |
| }, | |
| { | |
| "completion_length": 31.375, | |
| "epoch": 0.2995645863570392, | |
| "grad_norm": 2.0790046459772817, | |
| "kl": 0.72265625, | |
| "learning_rate": 8.213137379562485e-07, | |
| "loss": 0.0289, | |
| "reward": 0.580078125, | |
| "reward_std": 0.20096022635698318, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.462890625, | |
| "step": 258 | |
| }, | |
| { | |
| "completion_length": 74.29296875, | |
| "epoch": 0.3007256894049347, | |
| "grad_norm": 0.6853475598496406, | |
| "kl": 0.7080078125, | |
| "learning_rate": 8.198701368522412e-07, | |
| "loss": 0.0283, | |
| "reward": 0.54296875, | |
| "reward_std": 0.15424226596951485, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.46484375, | |
| "step": 259 | |
| }, | |
| { | |
| "completion_length": 38.55078125, | |
| "epoch": 0.3018867924528302, | |
| "grad_norm": 3.7197912494854255, | |
| "kl": 0.7197265625, | |
| "learning_rate": 8.184220078153768e-07, | |
| "loss": 0.0288, | |
| "reward": 0.564453125, | |
| "reward_std": 0.1538761891424656, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.470703125, | |
| "step": 260 | |
| }, | |
| { | |
| "completion_length": 35.63671875, | |
| "epoch": 0.3030478955007257, | |
| "grad_norm": 4.452860398190473, | |
| "kl": 0.72265625, | |
| "learning_rate": 8.169693713446959e-07, | |
| "loss": 0.0289, | |
| "reward": 0.599609375, | |
| "reward_std": 0.14453125, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.482421875, | |
| "step": 261 | |
| }, | |
| { | |
| "completion_length": 28.71484375, | |
| "epoch": 0.3042089985486212, | |
| "grad_norm": 0.39968494901525564, | |
| "kl": 0.73828125, | |
| "learning_rate": 8.155122480030452e-07, | |
| "loss": 0.0296, | |
| "reward": 0.61328125, | |
| "reward_std": 0.2245996594429016, | |
| "rewards/correctness_reward_func": 0.1328125, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 262 | |
| }, | |
| { | |
| "completion_length": 53.375, | |
| "epoch": 0.3053701015965167, | |
| "grad_norm": 2.851724228639477, | |
| "kl": 0.771484375, | |
| "learning_rate": 8.140506584167844e-07, | |
| "loss": 0.0309, | |
| "reward": 0.529296875, | |
| "reward_std": 0.1715945415198803, | |
| "rewards/correctness_reward_func": 0.0625, | |
| "rewards/strict_format_reward_func": 0.466796875, | |
| "step": 263 | |
| }, | |
| { | |
| "completion_length": 53.05078125, | |
| "epoch": 0.3065312046444122, | |
| "grad_norm": 1.7213671620214588, | |
| "kl": 0.6494140625, | |
| "learning_rate": 8.12584623275496e-07, | |
| "loss": 0.026, | |
| "reward": 0.544921875, | |
| "reward_std": 0.1769692748785019, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.466796875, | |
| "step": 264 | |
| }, | |
| { | |
| "completion_length": 50.8359375, | |
| "epoch": 0.3076923076923077, | |
| "grad_norm": 0.442713159312425, | |
| "kl": 0.6513671875, | |
| "learning_rate": 8.111141633316914e-07, | |
| "loss": 0.0261, | |
| "reward": 0.611328125, | |
| "reward_std": 0.18654337525367737, | |
| "rewards/correctness_reward_func": 0.140625, | |
| "rewards/strict_format_reward_func": 0.470703125, | |
| "step": 265 | |
| }, | |
| { | |
| "completion_length": 57.09375, | |
| "epoch": 0.30885341074020317, | |
| "grad_norm": 0.5370252752945665, | |
| "kl": 0.6982421875, | |
| "learning_rate": 8.096392994005176e-07, | |
| "loss": 0.0279, | |
| "reward": 0.5234375, | |
| "reward_std": 0.1663648895919323, | |
| "rewards/correctness_reward_func": 0.0703125, | |
| "rewards/strict_format_reward_func": 0.453125, | |
| "step": 266 | |
| }, | |
| { | |
| "completion_length": 38.22265625, | |
| "epoch": 0.3100145137880987, | |
| "grad_norm": 0.38559708214902666, | |
| "kl": 0.703125, | |
| "learning_rate": 8.08160052359462e-07, | |
| "loss": 0.0281, | |
| "reward": 0.607421875, | |
| "reward_std": 0.2017301730811596, | |
| "rewards/correctness_reward_func": 0.1328125, | |
| "rewards/strict_format_reward_func": 0.474609375, | |
| "step": 267 | |
| }, | |
| { | |
| "completion_length": 43.53515625, | |
| "epoch": 0.3111756168359942, | |
| "grad_norm": 0.8317974710954902, | |
| "kl": 0.7470703125, | |
| "learning_rate": 8.066764431480583e-07, | |
| "loss": 0.0299, | |
| "reward": 0.5703125, | |
| "reward_std": 0.1564590334892273, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 268 | |
| }, | |
| { | |
| "completion_length": 55.6484375, | |
| "epoch": 0.31233671988388967, | |
| "grad_norm": 0.674478273947861, | |
| "kl": 0.734375, | |
| "learning_rate": 8.051884927675877e-07, | |
| "loss": 0.0294, | |
| "reward": 0.5390625, | |
| "reward_std": 0.1821046955883503, | |
| "rewards/correctness_reward_func": 0.0859375, | |
| "rewards/strict_format_reward_func": 0.453125, | |
| "step": 269 | |
| }, | |
| { | |
| "completion_length": 30.5703125, | |
| "epoch": 0.3134978229317852, | |
| "grad_norm": 7.412108279999894, | |
| "kl": 0.759765625, | |
| "learning_rate": 8.036962222807837e-07, | |
| "loss": 0.0304, | |
| "reward": 0.58984375, | |
| "reward_std": 0.1655949391424656, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.47265625, | |
| "step": 270 | |
| }, | |
| { | |
| "completion_length": 27.45703125, | |
| "epoch": 0.3146589259796807, | |
| "grad_norm": 0.35968132821390736, | |
| "kl": 0.73828125, | |
| "learning_rate": 8.021996528115333e-07, | |
| "loss": 0.0295, | |
| "reward": 0.541015625, | |
| "reward_std": 0.1393449790775776, | |
| "rewards/correctness_reward_func": 0.0625, | |
| "rewards/strict_format_reward_func": 0.478515625, | |
| "step": 271 | |
| }, | |
| { | |
| "completion_length": 35.44140625, | |
| "epoch": 0.3158200290275762, | |
| "grad_norm": 0.23026737567150152, | |
| "kl": 0.7548828125, | |
| "learning_rate": 8.006988055445778e-07, | |
| "loss": 0.0301, | |
| "reward": 0.541015625, | |
| "reward_std": 0.07784454245120287, | |
| "rewards/correctness_reward_func": 0.0625, | |
| "rewards/strict_format_reward_func": 0.478515625, | |
| "step": 272 | |
| }, | |
| { | |
| "completion_length": 33.1484375, | |
| "epoch": 0.3169811320754717, | |
| "grad_norm": 0.31612172425026225, | |
| "kl": 0.7666015625, | |
| "learning_rate": 7.991937017252126e-07, | |
| "loss": 0.0307, | |
| "reward": 0.640625, | |
| "reward_std": 0.16107875108718872, | |
| "rewards/correctness_reward_func": 0.1640625, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 273 | |
| }, | |
| { | |
| "completion_length": 44.54296875, | |
| "epoch": 0.3181422351233672, | |
| "grad_norm": 0.6776325232505502, | |
| "kl": 0.763671875, | |
| "learning_rate": 7.976843626589874e-07, | |
| "loss": 0.0306, | |
| "reward": 0.671875, | |
| "reward_std": 0.17429219186306, | |
| "rewards/correctness_reward_func": 0.1953125, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 274 | |
| }, | |
| { | |
| "completion_length": 21.1328125, | |
| "epoch": 0.3193033381712627, | |
| "grad_norm": 0.3348417539199389, | |
| "kl": 0.8203125, | |
| "learning_rate": 7.961708097114039e-07, | |
| "loss": 0.0328, | |
| "reward": 0.642578125, | |
| "reward_std": 0.21827621944248676, | |
| "rewards/correctness_reward_func": 0.15625, | |
| "rewards/strict_format_reward_func": 0.486328125, | |
| "step": 275 | |
| }, | |
| { | |
| "completion_length": 29.984375, | |
| "epoch": 0.3204644412191582, | |
| "grad_norm": 0.19295177951085285, | |
| "kl": 0.7841796875, | |
| "learning_rate": 7.946530643076137e-07, | |
| "loss": 0.0314, | |
| "reward": 0.591796875, | |
| "reward_std": 0.1156984455883503, | |
| "rewards/correctness_reward_func": 0.109375, | |
| "rewards/strict_format_reward_func": 0.482421875, | |
| "step": 276 | |
| }, | |
| { | |
| "completion_length": 32.6484375, | |
| "epoch": 0.3216255442670537, | |
| "grad_norm": 0.24812441240225833, | |
| "kl": 0.7890625, | |
| "learning_rate": 7.931311479321143e-07, | |
| "loss": 0.0316, | |
| "reward": 0.642578125, | |
| "reward_std": 0.1479274332523346, | |
| "rewards/correctness_reward_func": 0.15625, | |
| "rewards/strict_format_reward_func": 0.486328125, | |
| "step": 277 | |
| }, | |
| { | |
| "completion_length": 51.703125, | |
| "epoch": 0.3227866473149492, | |
| "grad_norm": 0.24875270584616957, | |
| "kl": 0.7470703125, | |
| "learning_rate": 7.916050821284462e-07, | |
| "loss": 0.0299, | |
| "reward": 0.55078125, | |
| "reward_std": 0.1319277435541153, | |
| "rewards/correctness_reward_func": 0.0703125, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 278 | |
| }, | |
| { | |
| "completion_length": 25.44140625, | |
| "epoch": 0.3239477503628447, | |
| "grad_norm": 0.41045591923433544, | |
| "kl": 0.8203125, | |
| "learning_rate": 7.90074888498887e-07, | |
| "loss": 0.0329, | |
| "reward": 0.6640625, | |
| "reward_std": 0.20949183404445648, | |
| "rewards/correctness_reward_func": 0.1796875, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 279 | |
| }, | |
| { | |
| "completion_length": 33.265625, | |
| "epoch": 0.3251088534107402, | |
| "grad_norm": 0.5921301270100346, | |
| "kl": 0.794921875, | |
| "learning_rate": 7.885405887041459e-07, | |
| "loss": 0.0318, | |
| "reward": 0.556640625, | |
| "reward_std": 0.1345105916261673, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.478515625, | |
| "step": 280 | |
| }, | |
| { | |
| "completion_length": 43.90234375, | |
| "epoch": 0.3262699564586357, | |
| "grad_norm": 0.6987483029959047, | |
| "kl": 0.787109375, | |
| "learning_rate": 7.870022044630568e-07, | |
| "loss": 0.0315, | |
| "reward": 0.572265625, | |
| "reward_std": 0.1548914648592472, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.478515625, | |
| "step": 281 | |
| }, | |
| { | |
| "completion_length": 27.69140625, | |
| "epoch": 0.32743105950653123, | |
| "grad_norm": 0.9294871394333097, | |
| "kl": 0.78515625, | |
| "learning_rate": 7.854597575522717e-07, | |
| "loss": 0.0314, | |
| "reward": 0.625, | |
| "reward_std": 0.2526262328028679, | |
| "rewards/correctness_reward_func": 0.140625, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 282 | |
| }, | |
| { | |
| "completion_length": 46.27734375, | |
| "epoch": 0.3285921625544267, | |
| "grad_norm": 3.7732561147670802, | |
| "kl": 0.72265625, | |
| "learning_rate": 7.839132698059514e-07, | |
| "loss": 0.0289, | |
| "reward": 0.603515625, | |
| "reward_std": 0.1705949828028679, | |
| "rewards/correctness_reward_func": 0.109375, | |
| "rewards/strict_format_reward_func": 0.494140625, | |
| "step": 283 | |
| }, | |
| { | |
| "completion_length": 46.1953125, | |
| "epoch": 0.3297532656023222, | |
| "grad_norm": 0.3556241277311047, | |
| "kl": 0.76171875, | |
| "learning_rate": 7.82362763115457e-07, | |
| "loss": 0.0305, | |
| "reward": 0.58203125, | |
| "reward_std": 0.1598757915198803, | |
| "rewards/correctness_reward_func": 0.1015625, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 284 | |
| }, | |
| { | |
| "completion_length": 43.4921875, | |
| "epoch": 0.3309143686502177, | |
| "grad_norm": 0.3342258795715177, | |
| "kl": 0.7685546875, | |
| "learning_rate": 7.808082594290402e-07, | |
| "loss": 0.0308, | |
| "reward": 0.609375, | |
| "reward_std": 0.15624435618519783, | |
| "rewards/correctness_reward_func": 0.125, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 285 | |
| }, | |
| { | |
| "completion_length": 27.234375, | |
| "epoch": 0.3320754716981132, | |
| "grad_norm": 3.5943102994163274, | |
| "kl": 0.7783203125, | |
| "learning_rate": 7.792497807515316e-07, | |
| "loss": 0.0311, | |
| "reward": 0.64453125, | |
| "reward_std": 0.18605433031916618, | |
| "rewards/correctness_reward_func": 0.15625, | |
| "rewards/strict_format_reward_func": 0.48828125, | |
| "step": 286 | |
| }, | |
| { | |
| "completion_length": 43.7734375, | |
| "epoch": 0.3332365747460087, | |
| "grad_norm": 0.24602456316761495, | |
| "kl": 0.7333984375, | |
| "learning_rate": 7.776873491440307e-07, | |
| "loss": 0.0293, | |
| "reward": 0.60546875, | |
| "reward_std": 0.1486465372145176, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.48828125, | |
| "step": 287 | |
| }, | |
| { | |
| "completion_length": 42.15625, | |
| "epoch": 0.3343976777939042, | |
| "grad_norm": 0.600530335279195, | |
| "kl": 0.7314453125, | |
| "learning_rate": 7.761209867235923e-07, | |
| "loss": 0.0293, | |
| "reward": 0.609375, | |
| "reward_std": 0.2011621668934822, | |
| "rewards/correctness_reward_func": 0.125, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 288 | |
| }, | |
| { | |
| "completion_length": 39.6796875, | |
| "epoch": 0.3355587808417997, | |
| "grad_norm": 0.2995300813353396, | |
| "kl": 0.736328125, | |
| "learning_rate": 7.745507156629144e-07, | |
| "loss": 0.0294, | |
| "reward": 0.626953125, | |
| "reward_std": 0.1818399503827095, | |
| "rewards/correctness_reward_func": 0.140625, | |
| "rewards/strict_format_reward_func": 0.486328125, | |
| "step": 289 | |
| }, | |
| { | |
| "completion_length": 38.13671875, | |
| "epoch": 0.3367198838896952, | |
| "grad_norm": 1.044126937484855, | |
| "kl": 0.6904296875, | |
| "learning_rate": 7.729765581900235e-07, | |
| "loss": 0.0277, | |
| "reward": 0.544921875, | |
| "reward_std": 0.11239649448543787, | |
| "rewards/correctness_reward_func": 0.0625, | |
| "rewards/strict_format_reward_func": 0.482421875, | |
| "step": 290 | |
| }, | |
| { | |
| "completion_length": 49.296875, | |
| "epoch": 0.3378809869375907, | |
| "grad_norm": 0.2824327700655812, | |
| "kl": 0.703125, | |
| "learning_rate": 7.713985365879605e-07, | |
| "loss": 0.0281, | |
| "reward": 0.58203125, | |
| "reward_std": 0.20035592839121819, | |
| "rewards/correctness_reward_func": 0.1015625, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 291 | |
| }, | |
| { | |
| "completion_length": 65.84765625, | |
| "epoch": 0.33904208998548624, | |
| "grad_norm": 0.247824658119491, | |
| "kl": 0.716796875, | |
| "learning_rate": 7.698166731944653e-07, | |
| "loss": 0.0286, | |
| "reward": 0.533203125, | |
| "reward_std": 0.08323984593153, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.478515625, | |
| "step": 292 | |
| }, | |
| { | |
| "completion_length": 38.046875, | |
| "epoch": 0.3402031930333817, | |
| "grad_norm": 0.2895158117455258, | |
| "kl": 0.693359375, | |
| "learning_rate": 7.682309904016599e-07, | |
| "loss": 0.0277, | |
| "reward": 0.517578125, | |
| "reward_std": 0.0766359455883503, | |
| "rewards/correctness_reward_func": 0.03125, | |
| "rewards/strict_format_reward_func": 0.486328125, | |
| "step": 293 | |
| }, | |
| { | |
| "completion_length": 47.41796875, | |
| "epoch": 0.3413642960812772, | |
| "grad_norm": 0.4958137248441807, | |
| "kl": 0.7646484375, | |
| "learning_rate": 7.666415106557327e-07, | |
| "loss": 0.0305, | |
| "reward": 0.544921875, | |
| "reward_std": 0.11095601879060268, | |
| "rewards/correctness_reward_func": 0.0625, | |
| "rewards/strict_format_reward_func": 0.482421875, | |
| "step": 294 | |
| }, | |
| { | |
| "completion_length": 67.2265625, | |
| "epoch": 0.34252539912917274, | |
| "grad_norm": 0.4138009604439513, | |
| "kl": 0.6845703125, | |
| "learning_rate": 7.650482564566192e-07, | |
| "loss": 0.0274, | |
| "reward": 0.57421875, | |
| "reward_std": 0.2022402435541153, | |
| "rewards/correctness_reward_func": 0.1015625, | |
| "rewards/strict_format_reward_func": 0.47265625, | |
| "step": 295 | |
| }, | |
| { | |
| "completion_length": 35.2421875, | |
| "epoch": 0.3436865021770682, | |
| "grad_norm": 1.3963827489118958, | |
| "kl": 0.77734375, | |
| "learning_rate": 7.634512503576849e-07, | |
| "loss": 0.0311, | |
| "reward": 0.6171875, | |
| "reward_std": 0.109375, | |
| "rewards/correctness_reward_func": 0.1328125, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 296 | |
| }, | |
| { | |
| "completion_length": 42.29296875, | |
| "epoch": 0.3448476052249637, | |
| "grad_norm": 0.2378588696433431, | |
| "kl": 0.6171875, | |
| "learning_rate": 7.618505149654051e-07, | |
| "loss": 0.0247, | |
| "reward": 0.52734375, | |
| "reward_std": 0.0703125, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 297 | |
| }, | |
| { | |
| "completion_length": 55.24609375, | |
| "epoch": 0.34600870827285923, | |
| "grad_norm": 1.6488130875167075, | |
| "kl": 0.908203125, | |
| "learning_rate": 7.602460729390454e-07, | |
| "loss": 0.0363, | |
| "reward": 0.607421875, | |
| "reward_std": 0.2039383314549923, | |
| "rewards/correctness_reward_func": 0.1328125, | |
| "rewards/strict_format_reward_func": 0.474609375, | |
| "step": 298 | |
| }, | |
| { | |
| "completion_length": 57.0390625, | |
| "epoch": 0.3471698113207547, | |
| "grad_norm": 0.28158042067891265, | |
| "kl": 0.67578125, | |
| "learning_rate": 7.586379469903408e-07, | |
| "loss": 0.027, | |
| "reward": 0.5390625, | |
| "reward_std": 0.1252090372145176, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 299 | |
| }, | |
| { | |
| "completion_length": 70.46484375, | |
| "epoch": 0.3483309143686502, | |
| "grad_norm": 0.4201728799059877, | |
| "kl": 0.65234375, | |
| "learning_rate": 7.570261598831742e-07, | |
| "loss": 0.0261, | |
| "reward": 0.619140625, | |
| "reward_std": 0.19710255414247513, | |
| "rewards/correctness_reward_func": 0.140625, | |
| "rewards/strict_format_reward_func": 0.478515625, | |
| "step": 300 | |
| }, | |
| { | |
| "completion_length": 52.83984375, | |
| "epoch": 0.34949201741654573, | |
| "grad_norm": 0.2803079731494032, | |
| "kl": 0.6904296875, | |
| "learning_rate": 7.554107344332538e-07, | |
| "loss": 0.0276, | |
| "reward": 0.609375, | |
| "reward_std": 0.1758246347308159, | |
| "rewards/correctness_reward_func": 0.125, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 301 | |
| }, | |
| { | |
| "completion_length": 72.19921875, | |
| "epoch": 0.3506531204644412, | |
| "grad_norm": 0.32997263344841316, | |
| "kl": 0.6591796875, | |
| "learning_rate": 7.537916935077913e-07, | |
| "loss": 0.0264, | |
| "reward": 0.55078125, | |
| "reward_std": 0.1532718874514103, | |
| "rewards/correctness_reward_func": 0.0703125, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 302 | |
| }, | |
| { | |
| "completion_length": 59.51953125, | |
| "epoch": 0.3518142235123367, | |
| "grad_norm": 0.32482961945010463, | |
| "kl": 0.6728515625, | |
| "learning_rate": 7.521690600251764e-07, | |
| "loss": 0.0269, | |
| "reward": 0.556640625, | |
| "reward_std": 0.1361735537648201, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.478515625, | |
| "step": 303 | |
| }, | |
| { | |
| "completion_length": 71.43359375, | |
| "epoch": 0.3529753265602322, | |
| "grad_norm": 0.21926170187871932, | |
| "kl": 0.6142578125, | |
| "learning_rate": 7.505428569546542e-07, | |
| "loss": 0.0246, | |
| "reward": 0.578125, | |
| "reward_std": 0.1251305192708969, | |
| "rewards/correctness_reward_func": 0.1015625, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 304 | |
| }, | |
| { | |
| "completion_length": 83.47265625, | |
| "epoch": 0.35413642960812775, | |
| "grad_norm": 1.2095295337441825, | |
| "kl": 0.57421875, | |
| "learning_rate": 7.489131073159987e-07, | |
| "loss": 0.0229, | |
| "reward": 0.583984375, | |
| "reward_std": 0.22483637183904648, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.466796875, | |
| "step": 305 | |
| }, | |
| { | |
| "completion_length": 66.84765625, | |
| "epoch": 0.3552975326560232, | |
| "grad_norm": 0.2892214943510883, | |
| "kl": 0.55126953125, | |
| "learning_rate": 7.472798341791876e-07, | |
| "loss": 0.0221, | |
| "reward": 0.5546875, | |
| "reward_std": 0.20070216432213783, | |
| "rewards/correctness_reward_func": 0.0859375, | |
| "rewards/strict_format_reward_func": 0.46875, | |
| "step": 306 | |
| }, | |
| { | |
| "completion_length": 64.0703125, | |
| "epoch": 0.3564586357039187, | |
| "grad_norm": 0.27395677460271195, | |
| "kl": 0.533203125, | |
| "learning_rate": 7.456430606640757e-07, | |
| "loss": 0.0214, | |
| "reward": 0.533203125, | |
| "reward_std": 0.15057423803955317, | |
| "rewards/correctness_reward_func": 0.0625, | |
| "rewards/strict_format_reward_func": 0.470703125, | |
| "step": 307 | |
| }, | |
| { | |
| "completion_length": 93.12890625, | |
| "epoch": 0.35761973875181424, | |
| "grad_norm": 0.2623932543969406, | |
| "kl": 0.4482421875, | |
| "learning_rate": 7.440028099400676e-07, | |
| "loss": 0.0179, | |
| "reward": 0.53515625, | |
| "reward_std": 0.18847334384918213, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.45703125, | |
| "step": 308 | |
| }, | |
| { | |
| "completion_length": 96.703125, | |
| "epoch": 0.3587808417997097, | |
| "grad_norm": 0.421598782981462, | |
| "kl": 0.60205078125, | |
| "learning_rate": 7.423591052257893e-07, | |
| "loss": 0.024, | |
| "reward": 0.640625, | |
| "reward_std": 0.2437199391424656, | |
| "rewards/correctness_reward_func": 0.171875, | |
| "rewards/strict_format_reward_func": 0.46875, | |
| "step": 309 | |
| }, | |
| { | |
| "completion_length": 79.5, | |
| "epoch": 0.3599419448476052, | |
| "grad_norm": 0.3744214040179616, | |
| "kl": 0.51318359375, | |
| "learning_rate": 7.407119697887601e-07, | |
| "loss": 0.0205, | |
| "reward": 0.556640625, | |
| "reward_std": 0.07905314117670059, | |
| "rewards/correctness_reward_func": 0.0703125, | |
| "rewards/strict_format_reward_func": 0.486328125, | |
| "step": 310 | |
| }, | |
| { | |
| "completion_length": 91.90625, | |
| "epoch": 0.36110304789550074, | |
| "grad_norm": 0.29732270177076153, | |
| "kl": 0.4931640625, | |
| "learning_rate": 7.390614269450632e-07, | |
| "loss": 0.0197, | |
| "reward": 0.53515625, | |
| "reward_std": 0.19749262183904648, | |
| "rewards/correctness_reward_func": 0.0859375, | |
| "rewards/strict_format_reward_func": 0.44921875, | |
| "step": 311 | |
| }, | |
| { | |
| "completion_length": 90.94140625, | |
| "epoch": 0.3622641509433962, | |
| "grad_norm": 0.39324171683243425, | |
| "kl": 0.45166015625, | |
| "learning_rate": 7.374075000590155e-07, | |
| "loss": 0.0181, | |
| "reward": 0.5703125, | |
| "reward_std": 0.2068813145160675, | |
| "rewards/correctness_reward_func": 0.109375, | |
| "rewards/strict_format_reward_func": 0.4609375, | |
| "step": 312 | |
| }, | |
| { | |
| "completion_length": 88.10546875, | |
| "epoch": 0.3634252539912917, | |
| "grad_norm": 0.3999210759287133, | |
| "kl": 0.50537109375, | |
| "learning_rate": 7.357502125428358e-07, | |
| "loss": 0.0203, | |
| "reward": 0.611328125, | |
| "reward_std": 0.2668927013874054, | |
| "rewards/correctness_reward_func": 0.1484375, | |
| "rewards/strict_format_reward_func": 0.462890625, | |
| "step": 313 | |
| }, | |
| { | |
| "completion_length": 78.578125, | |
| "epoch": 0.36458635703918724, | |
| "grad_norm": 0.2934147726421681, | |
| "kl": 0.5, | |
| "learning_rate": 7.340895878563161e-07, | |
| "loss": 0.02, | |
| "reward": 0.59375, | |
| "reward_std": 0.20618987828493118, | |
| "rewards/correctness_reward_func": 0.125, | |
| "rewards/strict_format_reward_func": 0.46875, | |
| "step": 314 | |
| }, | |
| { | |
| "completion_length": 88.6484375, | |
| "epoch": 0.36574746008708275, | |
| "grad_norm": 0.304036568316934, | |
| "kl": 0.484375, | |
| "learning_rate": 7.324256495064866e-07, | |
| "loss": 0.0194, | |
| "reward": 0.595703125, | |
| "reward_std": 0.1582719311118126, | |
| "rewards/correctness_reward_func": 0.1328125, | |
| "rewards/strict_format_reward_func": 0.462890625, | |
| "step": 315 | |
| }, | |
| { | |
| "completion_length": 77.09375, | |
| "epoch": 0.3669085631349782, | |
| "grad_norm": 0.27730249987885874, | |
| "kl": 0.51171875, | |
| "learning_rate": 7.307584210472843e-07, | |
| "loss": 0.0205, | |
| "reward": 0.59375, | |
| "reward_std": 0.19424450024962425, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 316 | |
| }, | |
| { | |
| "completion_length": 79.24609375, | |
| "epoch": 0.36806966618287373, | |
| "grad_norm": 1.1443891203152459, | |
| "kl": 0.5205078125, | |
| "learning_rate": 7.290879260792202e-07, | |
| "loss": 0.0209, | |
| "reward": 0.599609375, | |
| "reward_std": 0.2074493207037449, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.482421875, | |
| "step": 317 | |
| }, | |
| { | |
| "completion_length": 85.109375, | |
| "epoch": 0.36923076923076925, | |
| "grad_norm": 0.8272363687978308, | |
| "kl": 0.53515625, | |
| "learning_rate": 7.274141882490435e-07, | |
| "loss": 0.0214, | |
| "reward": 0.623046875, | |
| "reward_std": 0.2066008560359478, | |
| "rewards/correctness_reward_func": 0.1640625, | |
| "rewards/strict_format_reward_func": 0.458984375, | |
| "step": 318 | |
| }, | |
| { | |
| "completion_length": 75.828125, | |
| "epoch": 0.3703918722786647, | |
| "grad_norm": 0.39409786758012594, | |
| "kl": 0.57666015625, | |
| "learning_rate": 7.257372312494087e-07, | |
| "loss": 0.0231, | |
| "reward": 0.677734375, | |
| "reward_std": 0.22956114634871483, | |
| "rewards/correctness_reward_func": 0.2109375, | |
| "rewards/strict_format_reward_func": 0.466796875, | |
| "step": 319 | |
| }, | |
| { | |
| "completion_length": 90.39453125, | |
| "epoch": 0.37155297532656023, | |
| "grad_norm": 0.5705020970860728, | |
| "kl": 0.5244140625, | |
| "learning_rate": 7.240570788185387e-07, | |
| "loss": 0.021, | |
| "reward": 0.64453125, | |
| "reward_std": 0.20728367194533348, | |
| "rewards/correctness_reward_func": 0.171875, | |
| "rewards/strict_format_reward_func": 0.47265625, | |
| "step": 320 | |
| }, | |
| { | |
| "completion_length": 112.0703125, | |
| "epoch": 0.37271407837445575, | |
| "grad_norm": 0.2780015737724446, | |
| "kl": 0.46728515625, | |
| "learning_rate": 7.223737547398897e-07, | |
| "loss": 0.0187, | |
| "reward": 0.552734375, | |
| "reward_std": 0.1929386854171753, | |
| "rewards/correctness_reward_func": 0.0859375, | |
| "rewards/strict_format_reward_func": 0.466796875, | |
| "step": 321 | |
| }, | |
| { | |
| "completion_length": 99.4140625, | |
| "epoch": 0.3738751814223512, | |
| "grad_norm": 3.609413827943461, | |
| "kl": 0.5478515625, | |
| "learning_rate": 7.206872828418146e-07, | |
| "loss": 0.0219, | |
| "reward": 0.580078125, | |
| "reward_std": 0.1582719311118126, | |
| "rewards/correctness_reward_func": 0.109375, | |
| "rewards/strict_format_reward_func": 0.470703125, | |
| "step": 322 | |
| }, | |
| { | |
| "completion_length": 78.84375, | |
| "epoch": 0.37503628447024673, | |
| "grad_norm": 0.3837761508066674, | |
| "kl": 0.51171875, | |
| "learning_rate": 7.189976869972247e-07, | |
| "loss": 0.0205, | |
| "reward": 0.568359375, | |
| "reward_std": 0.2117508314549923, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.474609375, | |
| "step": 323 | |
| }, | |
| { | |
| "completion_length": 101.32421875, | |
| "epoch": 0.37619738751814225, | |
| "grad_norm": 0.35836358831864257, | |
| "kl": 0.466796875, | |
| "learning_rate": 7.173049911232532e-07, | |
| "loss": 0.0187, | |
| "reward": 0.60546875, | |
| "reward_std": 0.2125641666352749, | |
| "rewards/correctness_reward_func": 0.1328125, | |
| "rewards/strict_format_reward_func": 0.47265625, | |
| "step": 324 | |
| }, | |
| { | |
| "completion_length": 93.78515625, | |
| "epoch": 0.37735849056603776, | |
| "grad_norm": 0.3954539999395301, | |
| "kl": 0.49169921875, | |
| "learning_rate": 7.156092191809151e-07, | |
| "loss": 0.0197, | |
| "reward": 0.6015625, | |
| "reward_std": 0.2178652472794056, | |
| "rewards/correctness_reward_func": 0.1328125, | |
| "rewards/strict_format_reward_func": 0.46875, | |
| "step": 325 | |
| }, | |
| { | |
| "completion_length": 95.578125, | |
| "epoch": 0.3785195936139332, | |
| "grad_norm": 0.30020285338127595, | |
| "kl": 0.4892578125, | |
| "learning_rate": 7.139103951747693e-07, | |
| "loss": 0.0195, | |
| "reward": 0.568359375, | |
| "reward_std": 0.1592714935541153, | |
| "rewards/correctness_reward_func": 0.1015625, | |
| "rewards/strict_format_reward_func": 0.466796875, | |
| "step": 326 | |
| }, | |
| { | |
| "completion_length": 94.33203125, | |
| "epoch": 0.37968069666182874, | |
| "grad_norm": 5.739385305144668, | |
| "kl": 0.4619140625, | |
| "learning_rate": 7.122085431525784e-07, | |
| "loss": 0.0185, | |
| "reward": 0.6171875, | |
| "reward_std": 0.2212977111339569, | |
| "rewards/correctness_reward_func": 0.1484375, | |
| "rewards/strict_format_reward_func": 0.46875, | |
| "step": 327 | |
| }, | |
| { | |
| "completion_length": 106.83984375, | |
| "epoch": 0.38084179970972426, | |
| "grad_norm": 0.2968677909998634, | |
| "kl": 0.45458984375, | |
| "learning_rate": 7.105036872049674e-07, | |
| "loss": 0.0182, | |
| "reward": 0.541015625, | |
| "reward_std": 0.17575840838253498, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.462890625, | |
| "step": 328 | |
| }, | |
| { | |
| "completion_length": 107.59765625, | |
| "epoch": 0.3820029027576197, | |
| "grad_norm": 0.3016049016709202, | |
| "kl": 0.46240234375, | |
| "learning_rate": 7.087958514650845e-07, | |
| "loss": 0.0185, | |
| "reward": 0.5703125, | |
| "reward_std": 0.1934281848371029, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.453125, | |
| "step": 329 | |
| }, | |
| { | |
| "completion_length": 86.56640625, | |
| "epoch": 0.38316400580551524, | |
| "grad_norm": 0.37897850223634566, | |
| "kl": 0.43017578125, | |
| "learning_rate": 7.070850601082577e-07, | |
| "loss": 0.0172, | |
| "reward": 0.578125, | |
| "reward_std": 0.1868242770433426, | |
| "rewards/correctness_reward_func": 0.1015625, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 330 | |
| }, | |
| { | |
| "completion_length": 115.5, | |
| "epoch": 0.38432510885341076, | |
| "grad_norm": 0.3956516881732845, | |
| "kl": 0.4375, | |
| "learning_rate": 7.053713373516536e-07, | |
| "loss": 0.0175, | |
| "reward": 0.572265625, | |
| "reward_std": 0.2532305307686329, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.455078125, | |
| "step": 331 | |
| }, | |
| { | |
| "completion_length": 92.05078125, | |
| "epoch": 0.3854862119013062, | |
| "grad_norm": 0.41709412462477324, | |
| "kl": 0.45458984375, | |
| "learning_rate": 7.036547074539347e-07, | |
| "loss": 0.0182, | |
| "reward": 0.5390625, | |
| "reward_std": 0.15031468868255615, | |
| "rewards/correctness_reward_func": 0.0703125, | |
| "rewards/strict_format_reward_func": 0.46875, | |
| "step": 332 | |
| }, | |
| { | |
| "completion_length": 99.34375, | |
| "epoch": 0.38664731494920174, | |
| "grad_norm": 0.26970793568837187, | |
| "kl": 0.4599609375, | |
| "learning_rate": 7.019351947149148e-07, | |
| "loss": 0.0184, | |
| "reward": 0.587890625, | |
| "reward_std": 0.2085794061422348, | |
| "rewards/correctness_reward_func": 0.125, | |
| "rewards/strict_format_reward_func": 0.462890625, | |
| "step": 333 | |
| }, | |
| { | |
| "completion_length": 91.33984375, | |
| "epoch": 0.38780841799709725, | |
| "grad_norm": 0.2837827283437701, | |
| "kl": 0.4306640625, | |
| "learning_rate": 7.002128234752166e-07, | |
| "loss": 0.0172, | |
| "reward": 0.578125, | |
| "reward_std": 0.1831898745149374, | |
| "rewards/correctness_reward_func": 0.1015625, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 334 | |
| }, | |
| { | |
| "completion_length": 94.125, | |
| "epoch": 0.3889695210449927, | |
| "grad_norm": 0.32889522420728007, | |
| "kl": 0.482421875, | |
| "learning_rate": 6.98487618115926e-07, | |
| "loss": 0.0193, | |
| "reward": 0.615234375, | |
| "reward_std": 0.1920539326965809, | |
| "rewards/correctness_reward_func": 0.140625, | |
| "rewards/strict_format_reward_func": 0.474609375, | |
| "step": 335 | |
| }, | |
| { | |
| "completion_length": 76.578125, | |
| "epoch": 0.39013062409288823, | |
| "grad_norm": 0.18090910204917746, | |
| "kl": 0.48193359375, | |
| "learning_rate": 6.967596030582477e-07, | |
| "loss": 0.0193, | |
| "reward": 0.564453125, | |
| "reward_std": 0.1259281411767006, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.486328125, | |
| "step": 336 | |
| }, | |
| { | |
| "completion_length": 109.40234375, | |
| "epoch": 0.39129172714078375, | |
| "grad_norm": 0.3527865653632661, | |
| "kl": 0.458984375, | |
| "learning_rate": 6.950288027631588e-07, | |
| "loss": 0.0184, | |
| "reward": 0.5703125, | |
| "reward_std": 0.16358720511198044, | |
| "rewards/correctness_reward_func": 0.109375, | |
| "rewards/strict_format_reward_func": 0.4609375, | |
| "step": 337 | |
| }, | |
| { | |
| "completion_length": 94.62890625, | |
| "epoch": 0.39245283018867927, | |
| "grad_norm": 0.32886929993323977, | |
| "kl": 0.50439453125, | |
| "learning_rate": 6.932952417310633e-07, | |
| "loss": 0.0202, | |
| "reward": 0.62109375, | |
| "reward_std": 0.20156452432274818, | |
| "rewards/correctness_reward_func": 0.1484375, | |
| "rewards/strict_format_reward_func": 0.47265625, | |
| "step": 338 | |
| }, | |
| { | |
| "completion_length": 62.5546875, | |
| "epoch": 0.39361393323657473, | |
| "grad_norm": 0.26246261432123036, | |
| "kl": 0.568359375, | |
| "learning_rate": 6.915589445014447e-07, | |
| "loss": 0.0228, | |
| "reward": 0.6015625, | |
| "reward_std": 0.1454593911767006, | |
| "rewards/correctness_reward_func": 0.125, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 339 | |
| }, | |
| { | |
| "completion_length": 71.5390625, | |
| "epoch": 0.39477503628447025, | |
| "grad_norm": 0.3297390493081332, | |
| "kl": 0.53564453125, | |
| "learning_rate": 6.89819935652519e-07, | |
| "loss": 0.0214, | |
| "reward": 0.521484375, | |
| "reward_std": 0.0485730916261673, | |
| "rewards/correctness_reward_func": 0.0390625, | |
| "rewards/strict_format_reward_func": 0.482421875, | |
| "step": 340 | |
| }, | |
| { | |
| "completion_length": 67.6875, | |
| "epoch": 0.39593613933236577, | |
| "grad_norm": 0.23108857562613203, | |
| "kl": 0.5341796875, | |
| "learning_rate": 6.88078239800886e-07, | |
| "loss": 0.0214, | |
| "reward": 0.52734375, | |
| "reward_std": 0.125, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 341 | |
| }, | |
| { | |
| "completion_length": 91.25, | |
| "epoch": 0.39709724238026123, | |
| "grad_norm": 0.25101702051238745, | |
| "kl": 0.48583984375, | |
| "learning_rate": 6.863338816011826e-07, | |
| "loss": 0.0194, | |
| "reward": 0.60546875, | |
| "reward_std": 0.18848609179258347, | |
| "rewards/correctness_reward_func": 0.125, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 342 | |
| }, | |
| { | |
| "completion_length": 75.06640625, | |
| "epoch": 0.39825834542815675, | |
| "grad_norm": 0.34327999543664667, | |
| "kl": 0.501953125, | |
| "learning_rate": 6.845868857457316e-07, | |
| "loss": 0.0201, | |
| "reward": 0.556640625, | |
| "reward_std": 0.12956623174250126, | |
| "rewards/correctness_reward_func": 0.0859375, | |
| "rewards/strict_format_reward_func": 0.470703125, | |
| "step": 343 | |
| }, | |
| { | |
| "completion_length": 88.91796875, | |
| "epoch": 0.39941944847605226, | |
| "grad_norm": 0.2879937533350012, | |
| "kl": 0.556640625, | |
| "learning_rate": 6.828372769641937e-07, | |
| "loss": 0.0222, | |
| "reward": 0.623046875, | |
| "reward_std": 0.18819626234471798, | |
| "rewards/correctness_reward_func": 0.1484375, | |
| "rewards/strict_format_reward_func": 0.474609375, | |
| "step": 344 | |
| }, | |
| { | |
| "completion_length": 86.1171875, | |
| "epoch": 0.4005805515239477, | |
| "grad_norm": 0.24685342319592915, | |
| "kl": 0.5478515625, | |
| "learning_rate": 6.810850800232171e-07, | |
| "loss": 0.0219, | |
| "reward": 0.54296875, | |
| "reward_std": 0.12476811930537224, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.46484375, | |
| "step": 345 | |
| }, | |
| { | |
| "completion_length": 73.68359375, | |
| "epoch": 0.40174165457184324, | |
| "grad_norm": 0.3263375483303881, | |
| "kl": 0.6025390625, | |
| "learning_rate": 6.793303197260864e-07, | |
| "loss": 0.0241, | |
| "reward": 0.513671875, | |
| "reward_std": 0.07893833611160517, | |
| "rewards/correctness_reward_func": 0.03125, | |
| "rewards/strict_format_reward_func": 0.482421875, | |
| "step": 346 | |
| }, | |
| { | |
| "completion_length": 83.9375, | |
| "epoch": 0.40290275761973876, | |
| "grad_norm": 0.25136463736995274, | |
| "kl": 0.54296875, | |
| "learning_rate": 6.775730209123722e-07, | |
| "loss": 0.0218, | |
| "reward": 0.6796875, | |
| "reward_std": 0.2113918662071228, | |
| "rewards/correctness_reward_func": 0.1953125, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 347 | |
| }, | |
| { | |
| "completion_length": 81.56640625, | |
| "epoch": 0.4040638606676343, | |
| "grad_norm": 0.2501886439839053, | |
| "kl": 0.5732421875, | |
| "learning_rate": 6.75813208457579e-07, | |
| "loss": 0.0229, | |
| "reward": 0.552734375, | |
| "reward_std": 0.1773136854171753, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.474609375, | |
| "step": 348 | |
| }, | |
| { | |
| "completion_length": 66.9765625, | |
| "epoch": 0.40522496371552974, | |
| "grad_norm": 0.5109927263281991, | |
| "kl": 0.6396484375, | |
| "learning_rate": 6.74050907272793e-07, | |
| "loss": 0.0256, | |
| "reward": 0.58203125, | |
| "reward_std": 0.1532718911767006, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.48828125, | |
| "step": 349 | |
| }, | |
| { | |
| "completion_length": 69.9609375, | |
| "epoch": 0.40638606676342526, | |
| "grad_norm": 0.22913115713166626, | |
| "kl": 0.623046875, | |
| "learning_rate": 6.722861423043305e-07, | |
| "loss": 0.0249, | |
| "reward": 0.54296875, | |
| "reward_std": 0.12951054889708757, | |
| "rewards/correctness_reward_func": 0.0625, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 350 | |
| }, | |
| { | |
| "completion_length": 57.07421875, | |
| "epoch": 0.4075471698113208, | |
| "grad_norm": 0.43448597905440733, | |
| "kl": 0.6708984375, | |
| "learning_rate": 6.70518938533383e-07, | |
| "loss": 0.0268, | |
| "reward": 0.765625, | |
| "reward_std": 0.21839812025427818, | |
| "rewards/correctness_reward_func": 0.28125, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 351 | |
| }, | |
| { | |
| "completion_length": 71.8984375, | |
| "epoch": 0.40870827285921624, | |
| "grad_norm": 0.5438789424715308, | |
| "kl": 0.6162109375, | |
| "learning_rate": 6.687493209756652e-07, | |
| "loss": 0.0247, | |
| "reward": 0.56640625, | |
| "reward_std": 0.2113555707037449, | |
| "rewards/correctness_reward_func": 0.1015625, | |
| "rewards/strict_format_reward_func": 0.46484375, | |
| "step": 352 | |
| }, | |
| { | |
| "completion_length": 78.9453125, | |
| "epoch": 0.40986937590711175, | |
| "grad_norm": 0.3643681813722083, | |
| "kl": 0.54638671875, | |
| "learning_rate": 6.669773146810598e-07, | |
| "loss": 0.0219, | |
| "reward": 0.595703125, | |
| "reward_std": 0.20702560991048813, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.478515625, | |
| "step": 353 | |
| }, | |
| { | |
| "completion_length": 74.796875, | |
| "epoch": 0.4110304789550073, | |
| "grad_norm": 0.24296888261067365, | |
| "kl": 0.58984375, | |
| "learning_rate": 6.652029447332641e-07, | |
| "loss": 0.0236, | |
| "reward": 0.619140625, | |
| "reward_std": 0.1514589935541153, | |
| "rewards/correctness_reward_func": 0.1484375, | |
| "rewards/strict_format_reward_func": 0.470703125, | |
| "step": 354 | |
| }, | |
| { | |
| "completion_length": 76.3203125, | |
| "epoch": 0.41219158200290273, | |
| "grad_norm": 0.3098307099636997, | |
| "kl": 0.6201171875, | |
| "learning_rate": 6.634262362494332e-07, | |
| "loss": 0.0248, | |
| "reward": 0.634765625, | |
| "reward_std": 0.1970105916261673, | |
| "rewards/correctness_reward_func": 0.15625, | |
| "rewards/strict_format_reward_func": 0.478515625, | |
| "step": 355 | |
| }, | |
| { | |
| "completion_length": 109.6796875, | |
| "epoch": 0.41335268505079825, | |
| "grad_norm": 0.22986280827840255, | |
| "kl": 0.53564453125, | |
| "learning_rate": 6.61647214379826e-07, | |
| "loss": 0.0214, | |
| "reward": 0.548828125, | |
| "reward_std": 0.1738184168934822, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.470703125, | |
| "step": 356 | |
| }, | |
| { | |
| "completion_length": 101.62890625, | |
| "epoch": 0.41451378809869377, | |
| "grad_norm": 0.3167144536185835, | |
| "kl": 0.564453125, | |
| "learning_rate": 6.598659043074486e-07, | |
| "loss": 0.0225, | |
| "reward": 0.546875, | |
| "reward_std": 0.17615367099642754, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.46875, | |
| "step": 357 | |
| }, | |
| { | |
| "completion_length": 76.7734375, | |
| "epoch": 0.4156748911465893, | |
| "grad_norm": 0.24303041290839197, | |
| "kl": 0.54638671875, | |
| "learning_rate": 6.580823312476975e-07, | |
| "loss": 0.0218, | |
| "reward": 0.521484375, | |
| "reward_std": 0.10546875, | |
| "rewards/correctness_reward_func": 0.0390625, | |
| "rewards/strict_format_reward_func": 0.482421875, | |
| "step": 358 | |
| }, | |
| { | |
| "completion_length": 125.3671875, | |
| "epoch": 0.41683599419448475, | |
| "grad_norm": 0.7081131603026585, | |
| "kl": 0.52880859375, | |
| "learning_rate": 6.562965204480037e-07, | |
| "loss": 0.0211, | |
| "reward": 0.6171875, | |
| "reward_std": 0.2204914726316929, | |
| "rewards/correctness_reward_func": 0.1484375, | |
| "rewards/strict_format_reward_func": 0.46875, | |
| "step": 359 | |
| }, | |
| { | |
| "completion_length": 82.41796875, | |
| "epoch": 0.41799709724238027, | |
| "grad_norm": 0.4357938018159004, | |
| "kl": 0.54443359375, | |
| "learning_rate": 6.545084971874736e-07, | |
| "loss": 0.0218, | |
| "reward": 0.646484375, | |
| "reward_std": 0.19245435670018196, | |
| "rewards/correctness_reward_func": 0.1796875, | |
| "rewards/strict_format_reward_func": 0.466796875, | |
| "step": 360 | |
| }, | |
| { | |
| "completion_length": 105.57421875, | |
| "epoch": 0.4191582002902758, | |
| "grad_norm": 0.4400020096023354, | |
| "kl": 0.45703125, | |
| "learning_rate": 6.527182867765332e-07, | |
| "loss": 0.0183, | |
| "reward": 0.62890625, | |
| "reward_std": 0.2650640867650509, | |
| "rewards/correctness_reward_func": 0.15625, | |
| "rewards/strict_format_reward_func": 0.47265625, | |
| "step": 361 | |
| }, | |
| { | |
| "completion_length": 101.62109375, | |
| "epoch": 0.42031930333817125, | |
| "grad_norm": 0.4513218352418108, | |
| "kl": 0.5908203125, | |
| "learning_rate": 6.50925914556568e-07, | |
| "loss": 0.0236, | |
| "reward": 0.591796875, | |
| "reward_std": 0.20470601320266724, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.474609375, | |
| "step": 362 | |
| }, | |
| { | |
| "completion_length": 114.00390625, | |
| "epoch": 0.42148040638606676, | |
| "grad_norm": 0.2773760092777871, | |
| "kl": 0.55859375, | |
| "learning_rate": 6.491314058995652e-07, | |
| "loss": 0.0224, | |
| "reward": 0.61328125, | |
| "reward_std": 0.15206329338252544, | |
| "rewards/correctness_reward_func": 0.140625, | |
| "rewards/strict_format_reward_func": 0.47265625, | |
| "step": 363 | |
| }, | |
| { | |
| "completion_length": 142.7265625, | |
| "epoch": 0.4226415094339623, | |
| "grad_norm": 0.27616972801783, | |
| "kl": 0.50244140625, | |
| "learning_rate": 6.473347862077551e-07, | |
| "loss": 0.0201, | |
| "reward": 0.54296875, | |
| "reward_std": 0.1957305707037449, | |
| "rewards/correctness_reward_func": 0.0859375, | |
| "rewards/strict_format_reward_func": 0.45703125, | |
| "step": 364 | |
| }, | |
| { | |
| "completion_length": 94.98046875, | |
| "epoch": 0.42380261248185774, | |
| "grad_norm": 0.18555372496203534, | |
| "kl": 0.5654296875, | |
| "learning_rate": 6.455360809132497e-07, | |
| "loss": 0.0226, | |
| "reward": 0.576171875, | |
| "reward_std": 0.06310429889708757, | |
| "rewards/correctness_reward_func": 0.1015625, | |
| "rewards/strict_format_reward_func": 0.474609375, | |
| "step": 365 | |
| }, | |
| { | |
| "completion_length": 87.3515625, | |
| "epoch": 0.42496371552975326, | |
| "grad_norm": 0.4239263725202524, | |
| "kl": 0.57421875, | |
| "learning_rate": 6.437353154776848e-07, | |
| "loss": 0.023, | |
| "reward": 0.564453125, | |
| "reward_std": 0.14582546427845955, | |
| "rewards/correctness_reward_func": 0.1015625, | |
| "rewards/strict_format_reward_func": 0.462890625, | |
| "step": 366 | |
| }, | |
| { | |
| "completion_length": 94.11328125, | |
| "epoch": 0.4261248185776488, | |
| "grad_norm": 0.5104772858886346, | |
| "kl": 0.5341796875, | |
| "learning_rate": 6.41932515391858e-07, | |
| "loss": 0.0214, | |
| "reward": 0.583984375, | |
| "reward_std": 0.21658522635698318, | |
| "rewards/correctness_reward_func": 0.109375, | |
| "rewards/strict_format_reward_func": 0.474609375, | |
| "step": 367 | |
| }, | |
| { | |
| "completion_length": 71.8828125, | |
| "epoch": 0.42728592162554424, | |
| "grad_norm": 0.37470581187227664, | |
| "kl": 0.638671875, | |
| "learning_rate": 6.401277061753688e-07, | |
| "loss": 0.0255, | |
| "reward": 0.66796875, | |
| "reward_std": 0.2147723287343979, | |
| "rewards/correctness_reward_func": 0.1953125, | |
| "rewards/strict_format_reward_func": 0.47265625, | |
| "step": 368 | |
| }, | |
| { | |
| "completion_length": 98.078125, | |
| "epoch": 0.42844702467343976, | |
| "grad_norm": 0.2383430376592154, | |
| "kl": 0.5888671875, | |
| "learning_rate": 6.383209133762568e-07, | |
| "loss": 0.0236, | |
| "reward": 0.517578125, | |
| "reward_std": 0.11481368914246559, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.470703125, | |
| "step": 369 | |
| }, | |
| { | |
| "completion_length": 78.734375, | |
| "epoch": 0.4296081277213353, | |
| "grad_norm": 0.2147373893634736, | |
| "kl": 0.6044921875, | |
| "learning_rate": 6.365121625706404e-07, | |
| "loss": 0.0242, | |
| "reward": 0.580078125, | |
| "reward_std": 0.1355258673429489, | |
| "rewards/correctness_reward_func": 0.1015625, | |
| "rewards/strict_format_reward_func": 0.478515625, | |
| "step": 370 | |
| }, | |
| { | |
| "completion_length": 126.3359375, | |
| "epoch": 0.4307692307692308, | |
| "grad_norm": 0.29213327316917803, | |
| "kl": 0.57421875, | |
| "learning_rate": 6.347014793623548e-07, | |
| "loss": 0.023, | |
| "reward": 0.625, | |
| "reward_std": 0.1084902435541153, | |
| "rewards/correctness_reward_func": 0.1484375, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 371 | |
| }, | |
| { | |
| "completion_length": 82.0078125, | |
| "epoch": 0.43193033381712626, | |
| "grad_norm": 0.2485460820708867, | |
| "kl": 0.5703125, | |
| "learning_rate": 6.328888893825887e-07, | |
| "loss": 0.0228, | |
| "reward": 0.64453125, | |
| "reward_std": 0.1776098608970642, | |
| "rewards/correctness_reward_func": 0.1640625, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 372 | |
| }, | |
| { | |
| "completion_length": 59.90625, | |
| "epoch": 0.4330914368650218, | |
| "grad_norm": 0.2750923880024312, | |
| "kl": 0.58984375, | |
| "learning_rate": 6.31074418289523e-07, | |
| "loss": 0.0236, | |
| "reward": 0.607421875, | |
| "reward_std": 0.17719027400016785, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.490234375, | |
| "step": 373 | |
| }, | |
| { | |
| "completion_length": 110.38671875, | |
| "epoch": 0.4342525399129173, | |
| "grad_norm": 0.25779538772802885, | |
| "kl": 0.5966796875, | |
| "learning_rate": 6.292580917679664e-07, | |
| "loss": 0.0239, | |
| "reward": 0.583984375, | |
| "reward_std": 0.1581571288406849, | |
| "rewards/correctness_reward_func": 0.1015625, | |
| "rewards/strict_format_reward_func": 0.482421875, | |
| "step": 374 | |
| }, | |
| { | |
| "completion_length": 96.2578125, | |
| "epoch": 0.43541364296081275, | |
| "grad_norm": 0.5070325029886389, | |
| "kl": 0.6025390625, | |
| "learning_rate": 6.274399355289923e-07, | |
| "loss": 0.0241, | |
| "reward": 0.6015625, | |
| "reward_std": 0.18991719186306, | |
| "rewards/correctness_reward_func": 0.125, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 375 | |
| }, | |
| { | |
| "completion_length": 111.56640625, | |
| "epoch": 0.43657474600870827, | |
| "grad_norm": 0.31442374214756924, | |
| "kl": 0.55810546875, | |
| "learning_rate": 6.256199753095745e-07, | |
| "loss": 0.0223, | |
| "reward": 0.5703125, | |
| "reward_std": 0.1655949391424656, | |
| "rewards/correctness_reward_func": 0.1015625, | |
| "rewards/strict_format_reward_func": 0.46875, | |
| "step": 376 | |
| }, | |
| { | |
| "completion_length": 93.6640625, | |
| "epoch": 0.4377358490566038, | |
| "grad_norm": 6.9371543727247005, | |
| "kl": 0.609375, | |
| "learning_rate": 6.237982368722231e-07, | |
| "loss": 0.0244, | |
| "reward": 0.671875, | |
| "reward_std": 0.3408661223948002, | |
| "rewards/correctness_reward_func": 0.203125, | |
| "rewards/strict_format_reward_func": 0.46875, | |
| "step": 377 | |
| }, | |
| { | |
| "completion_length": 106.3046875, | |
| "epoch": 0.43889695210449925, | |
| "grad_norm": 0.48230199921643463, | |
| "kl": 0.5869140625, | |
| "learning_rate": 6.219747460046202e-07, | |
| "loss": 0.0235, | |
| "reward": 0.666015625, | |
| "reward_std": 0.17466461658477783, | |
| "rewards/correctness_reward_func": 0.1953125, | |
| "rewards/strict_format_reward_func": 0.470703125, | |
| "step": 378 | |
| }, | |
| { | |
| "completion_length": 99.83984375, | |
| "epoch": 0.44005805515239477, | |
| "grad_norm": 0.25761065472308364, | |
| "kl": 0.54541015625, | |
| "learning_rate": 6.201495285192542e-07, | |
| "loss": 0.0218, | |
| "reward": 0.568359375, | |
| "reward_std": 0.15420002862811089, | |
| "rewards/correctness_reward_func": 0.0859375, | |
| "rewards/strict_format_reward_func": 0.482421875, | |
| "step": 379 | |
| }, | |
| { | |
| "completion_length": 87.4921875, | |
| "epoch": 0.4412191582002903, | |
| "grad_norm": 1.029791050074705, | |
| "kl": 0.5869140625, | |
| "learning_rate": 6.183226102530546e-07, | |
| "loss": 0.0235, | |
| "reward": 0.587890625, | |
| "reward_std": 0.2398294061422348, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.470703125, | |
| "step": 380 | |
| }, | |
| { | |
| "completion_length": 83.20703125, | |
| "epoch": 0.4423802612481858, | |
| "grad_norm": 0.2982447065274822, | |
| "kl": 0.6044921875, | |
| "learning_rate": 6.164940170670265e-07, | |
| "loss": 0.0242, | |
| "reward": 0.671875, | |
| "reward_std": 0.2199743092060089, | |
| "rewards/correctness_reward_func": 0.1875, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 381 | |
| }, | |
| { | |
| "completion_length": 86.09765625, | |
| "epoch": 0.44354136429608126, | |
| "grad_norm": 0.9321126474041298, | |
| "kl": 0.6240234375, | |
| "learning_rate": 6.146637748458848e-07, | |
| "loss": 0.025, | |
| "reward": 0.61328125, | |
| "reward_std": 0.18747715651988983, | |
| "rewards/correctness_reward_func": 0.1328125, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 382 | |
| }, | |
| { | |
| "completion_length": 134.3671875, | |
| "epoch": 0.4447024673439768, | |
| "grad_norm": 0.45890211100180406, | |
| "kl": 0.5546875, | |
| "learning_rate": 6.128319094976868e-07, | |
| "loss": 0.0222, | |
| "reward": 0.548828125, | |
| "reward_std": 0.15234375, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.470703125, | |
| "step": 383 | |
| }, | |
| { | |
| "completion_length": 81.24609375, | |
| "epoch": 0.4458635703918723, | |
| "grad_norm": 2.248668131010746, | |
| "kl": 0.62890625, | |
| "learning_rate": 6.10998446953466e-07, | |
| "loss": 0.0252, | |
| "reward": 0.6171875, | |
| "reward_std": 0.21718952059745789, | |
| "rewards/correctness_reward_func": 0.140625, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 384 | |
| }, | |
| { | |
| "completion_length": 146.41796875, | |
| "epoch": 0.44702467343976776, | |
| "grad_norm": 0.2598233426911348, | |
| "kl": 0.6171875, | |
| "learning_rate": 6.091634131668651e-07, | |
| "loss": 0.0247, | |
| "reward": 0.615234375, | |
| "reward_std": 0.19898167252540588, | |
| "rewards/correctness_reward_func": 0.1484375, | |
| "rewards/strict_format_reward_func": 0.466796875, | |
| "step": 385 | |
| }, | |
| { | |
| "completion_length": 120.53125, | |
| "epoch": 0.4481857764876633, | |
| "grad_norm": 0.5394807067049577, | |
| "kl": 0.58203125, | |
| "learning_rate": 6.073268341137693e-07, | |
| "loss": 0.0233, | |
| "reward": 0.640625, | |
| "reward_std": 0.18835671991109848, | |
| "rewards/correctness_reward_func": 0.1640625, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 386 | |
| }, | |
| { | |
| "completion_length": 113.140625, | |
| "epoch": 0.4493468795355588, | |
| "grad_norm": 0.24295613961307327, | |
| "kl": 0.552734375, | |
| "learning_rate": 6.05488735791937e-07, | |
| "loss": 0.0221, | |
| "reward": 0.572265625, | |
| "reward_std": 0.21658522635698318, | |
| "rewards/correctness_reward_func": 0.1015625, | |
| "rewards/strict_format_reward_func": 0.470703125, | |
| "step": 387 | |
| }, | |
| { | |
| "completion_length": 120.71484375, | |
| "epoch": 0.45050798258345426, | |
| "grad_norm": 0.2607770576009339, | |
| "kl": 0.48095703125, | |
| "learning_rate": 6.036491442206329e-07, | |
| "loss": 0.0192, | |
| "reward": 0.587890625, | |
| "reward_std": 0.20744071155786514, | |
| "rewards/correctness_reward_func": 0.125, | |
| "rewards/strict_format_reward_func": 0.462890625, | |
| "step": 388 | |
| }, | |
| { | |
| "completion_length": 99.9453125, | |
| "epoch": 0.4516690856313498, | |
| "grad_norm": 16.372003186126687, | |
| "kl": 0.56005859375, | |
| "learning_rate": 6.018080854402599e-07, | |
| "loss": 0.0224, | |
| "reward": 0.669921875, | |
| "reward_std": 0.22427431121468544, | |
| "rewards/correctness_reward_func": 0.1953125, | |
| "rewards/strict_format_reward_func": 0.474609375, | |
| "step": 389 | |
| }, | |
| { | |
| "completion_length": 194.45703125, | |
| "epoch": 0.4528301886792453, | |
| "grad_norm": 0.3000541660710553, | |
| "kl": 0.4609375, | |
| "learning_rate": 5.999655855119892e-07, | |
| "loss": 0.0184, | |
| "reward": 0.634765625, | |
| "reward_std": 0.23597173392772675, | |
| "rewards/correctness_reward_func": 0.1875, | |
| "rewards/strict_format_reward_func": 0.447265625, | |
| "step": 390 | |
| }, | |
| { | |
| "completion_length": 184.12109375, | |
| "epoch": 0.4539912917271408, | |
| "grad_norm": 0.3095778168804227, | |
| "kl": 0.51953125, | |
| "learning_rate": 5.981216705173929e-07, | |
| "loss": 0.0208, | |
| "reward": 0.62890625, | |
| "reward_std": 0.2303973250091076, | |
| "rewards/correctness_reward_func": 0.1796875, | |
| "rewards/strict_format_reward_func": 0.44921875, | |
| "step": 391 | |
| }, | |
| { | |
| "completion_length": 119.00390625, | |
| "epoch": 0.4551523947750363, | |
| "grad_norm": 0.2428265662631221, | |
| "kl": 0.6015625, | |
| "learning_rate": 5.96276366558074e-07, | |
| "loss": 0.0241, | |
| "reward": 0.6171875, | |
| "reward_std": 0.22125915624201298, | |
| "rewards/correctness_reward_func": 0.140625, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 392 | |
| }, | |
| { | |
| "completion_length": 127.8125, | |
| "epoch": 0.4563134978229318, | |
| "grad_norm": 1.0186163125450653, | |
| "kl": 0.46484375, | |
| "learning_rate": 5.944296997552967e-07, | |
| "loss": 0.0186, | |
| "reward": 0.708984375, | |
| "reward_std": 0.2689474932849407, | |
| "rewards/correctness_reward_func": 0.2421875, | |
| "rewards/strict_format_reward_func": 0.466796875, | |
| "step": 393 | |
| }, | |
| { | |
| "completion_length": 126.64453125, | |
| "epoch": 0.4574746008708273, | |
| "grad_norm": 0.2738447873229587, | |
| "kl": 0.47021484375, | |
| "learning_rate": 5.925816962496174e-07, | |
| "loss": 0.0188, | |
| "reward": 0.666015625, | |
| "reward_std": 0.2515758164227009, | |
| "rewards/correctness_reward_func": 0.1953125, | |
| "rewards/strict_format_reward_func": 0.470703125, | |
| "step": 394 | |
| }, | |
| { | |
| "completion_length": 120.52734375, | |
| "epoch": 0.45863570391872277, | |
| "grad_norm": 0.2081853966183399, | |
| "kl": 0.52197265625, | |
| "learning_rate": 5.907323822005139e-07, | |
| "loss": 0.0209, | |
| "reward": 0.6640625, | |
| "reward_std": 0.16162246465682983, | |
| "rewards/correctness_reward_func": 0.1875, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 395 | |
| }, | |
| { | |
| "completion_length": 126.0625, | |
| "epoch": 0.4597968069666183, | |
| "grad_norm": 0.41377371550793696, | |
| "kl": 0.5361328125, | |
| "learning_rate": 5.888817837860156e-07, | |
| "loss": 0.0214, | |
| "reward": 0.560546875, | |
| "reward_std": 0.13253204245120287, | |
| "rewards/correctness_reward_func": 0.1015625, | |
| "rewards/strict_format_reward_func": 0.458984375, | |
| "step": 396 | |
| }, | |
| { | |
| "completion_length": 120.0703125, | |
| "epoch": 0.4609579100145138, | |
| "grad_norm": 0.2171941952922665, | |
| "kl": 0.52978515625, | |
| "learning_rate": 5.870299272023329e-07, | |
| "loss": 0.0212, | |
| "reward": 0.5234375, | |
| "reward_std": 0.1531570851802826, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.46875, | |
| "step": 397 | |
| }, | |
| { | |
| "completion_length": 141.9453125, | |
| "epoch": 0.46211901306240927, | |
| "grad_norm": 3.0625655639219964, | |
| "kl": 0.6064453125, | |
| "learning_rate": 5.851768386634862e-07, | |
| "loss": 0.0242, | |
| "reward": 0.5390625, | |
| "reward_std": 0.19876068457961082, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.4453125, | |
| "step": 398 | |
| }, | |
| { | |
| "completion_length": 90.546875, | |
| "epoch": 0.4632801161103048, | |
| "grad_norm": 1.1905377467292104, | |
| "kl": 0.6103515625, | |
| "learning_rate": 5.833225444009347e-07, | |
| "loss": 0.0244, | |
| "reward": 0.5703125, | |
| "reward_std": 0.1565895602107048, | |
| "rewards/correctness_reward_func": 0.1015625, | |
| "rewards/strict_format_reward_func": 0.46875, | |
| "step": 399 | |
| }, | |
| { | |
| "completion_length": 123.75, | |
| "epoch": 0.4644412191582003, | |
| "grad_norm": 0.23261666216565308, | |
| "kl": 0.5498046875, | |
| "learning_rate": 5.814670706632053e-07, | |
| "loss": 0.022, | |
| "reward": 0.533203125, | |
| "reward_std": 0.11901611275970936, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.478515625, | |
| "step": 400 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 861, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 100, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": false | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 4, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |