| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.23222060957910015, | |
| "eval_steps": 500, | |
| "global_step": 200, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "completion_length": 724.58984375, | |
| "epoch": 0.0011611030478955006, | |
| "grad_norm": 0.7112110585443041, | |
| "kl": 0.0009012222290039062, | |
| "learning_rate": 3.846153846153846e-08, | |
| "loss": 0.0, | |
| "reward": 0.046875, | |
| "reward_std": 0.06733439117670059, | |
| "rewards/correctness_reward_func": 0.0390625, | |
| "rewards/strict_format_reward_func": 0.0078125, | |
| "step": 1 | |
| }, | |
| { | |
| "completion_length": 708.56640625, | |
| "epoch": 0.0023222060957910013, | |
| "grad_norm": 0.12068406687948798, | |
| "kl": 0.000453948974609375, | |
| "learning_rate": 7.692307692307692e-08, | |
| "loss": 0.0, | |
| "reward": 0.033203125, | |
| "reward_std": 0.06640625, | |
| "rewards/correctness_reward_func": 0.03125, | |
| "rewards/strict_format_reward_func": 0.001953125, | |
| "step": 2 | |
| }, | |
| { | |
| "completion_length": 742.65625, | |
| "epoch": 0.0034833091436865023, | |
| "grad_norm": 0.7976659386050398, | |
| "kl": 0.0007343292236328125, | |
| "learning_rate": 1.1538461538461539e-07, | |
| "loss": 0.0, | |
| "reward": 0.0234375, | |
| "reward_std": 0.046875, | |
| "rewards/correctness_reward_func": 0.015625, | |
| "rewards/strict_format_reward_func": 0.0078125, | |
| "step": 3 | |
| }, | |
| { | |
| "completion_length": 808.21875, | |
| "epoch": 0.0046444121915820025, | |
| "grad_norm": 0.40112277888275455, | |
| "kl": 0.00063323974609375, | |
| "learning_rate": 1.5384615384615385e-07, | |
| "loss": 0.0, | |
| "reward": 0.060546875, | |
| "reward_std": 0.1078859455883503, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.005859375, | |
| "step": 4 | |
| }, | |
| { | |
| "completion_length": 654.234375, | |
| "epoch": 0.005805515239477504, | |
| "grad_norm": 0.39725912723012635, | |
| "kl": 0.0004980564117431641, | |
| "learning_rate": 1.9230769230769231e-07, | |
| "loss": 0.0, | |
| "reward": 0.044921875, | |
| "reward_std": 0.08984375, | |
| "rewards/correctness_reward_func": 0.03125, | |
| "rewards/strict_format_reward_func": 0.013671875, | |
| "step": 5 | |
| }, | |
| { | |
| "completion_length": 772.68359375, | |
| "epoch": 0.006966618287373005, | |
| "grad_norm": 0.14294736924830026, | |
| "kl": 0.0004153251647949219, | |
| "learning_rate": 2.3076923076923078e-07, | |
| "loss": 0.0, | |
| "reward": 0.046875, | |
| "reward_std": 0.0805421955883503, | |
| "rewards/correctness_reward_func": 0.0390625, | |
| "rewards/strict_format_reward_func": 0.0078125, | |
| "step": 6 | |
| }, | |
| { | |
| "completion_length": 745.203125, | |
| "epoch": 0.008127721335268505, | |
| "grad_norm": 0.16815060965909404, | |
| "kl": 0.0009188652038574219, | |
| "learning_rate": 2.692307692307692e-07, | |
| "loss": 0.0, | |
| "reward": 0.021484375, | |
| "reward_std": 0.029760945588350296, | |
| "rewards/correctness_reward_func": 0.015625, | |
| "rewards/strict_format_reward_func": 0.005859375, | |
| "step": 7 | |
| }, | |
| { | |
| "completion_length": 744.265625, | |
| "epoch": 0.009288824383164005, | |
| "grad_norm": 12.779442066715234, | |
| "kl": 0.00357818603515625, | |
| "learning_rate": 3.076923076923077e-07, | |
| "loss": 0.0001, | |
| "reward": 0.03125, | |
| "reward_std": 0.049292195588350296, | |
| "rewards/correctness_reward_func": 0.03125, | |
| "rewards/strict_format_reward_func": 0.0, | |
| "step": 8 | |
| }, | |
| { | |
| "completion_length": 704.51953125, | |
| "epoch": 0.010449927431059507, | |
| "grad_norm": 0.2475578183097461, | |
| "kl": 0.0035419464111328125, | |
| "learning_rate": 3.461538461538461e-07, | |
| "loss": 0.0001, | |
| "reward": 0.060546875, | |
| "reward_std": 0.1078859455883503, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.005859375, | |
| "step": 9 | |
| }, | |
| { | |
| "completion_length": 759.2890625, | |
| "epoch": 0.011611030478955007, | |
| "grad_norm": 0.3357083329970443, | |
| "kl": 0.00310516357421875, | |
| "learning_rate": 3.8461538461538463e-07, | |
| "loss": 0.0001, | |
| "reward": 0.00390625, | |
| "reward_std": 0.0078125, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.00390625, | |
| "step": 10 | |
| }, | |
| { | |
| "completion_length": 746.0703125, | |
| "epoch": 0.012772133526850507, | |
| "grad_norm": 0.2657165785401948, | |
| "kl": 0.00334930419921875, | |
| "learning_rate": 4.2307692307692304e-07, | |
| "loss": 0.0001, | |
| "reward": 0.017578125, | |
| "reward_std": 0.03515625, | |
| "rewards/correctness_reward_func": 0.015625, | |
| "rewards/strict_format_reward_func": 0.001953125, | |
| "step": 11 | |
| }, | |
| { | |
| "completion_length": 702.21875, | |
| "epoch": 0.01393323657474601, | |
| "grad_norm": 1.2656634353615759, | |
| "kl": 0.0019159317016601562, | |
| "learning_rate": 4.6153846153846156e-07, | |
| "loss": 0.0001, | |
| "reward": 0.033203125, | |
| "reward_std": 0.06640625, | |
| "rewards/correctness_reward_func": 0.03125, | |
| "rewards/strict_format_reward_func": 0.001953125, | |
| "step": 12 | |
| }, | |
| { | |
| "completion_length": 735.78515625, | |
| "epoch": 0.01509433962264151, | |
| "grad_norm": 0.5357650576255997, | |
| "kl": 0.005596160888671875, | |
| "learning_rate": 5e-07, | |
| "loss": 0.0002, | |
| "reward": 0.06640625, | |
| "reward_std": 0.1280700732022524, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.01171875, | |
| "step": 13 | |
| }, | |
| { | |
| "completion_length": 802.51953125, | |
| "epoch": 0.01625544267053701, | |
| "grad_norm": 1.9685839556774372, | |
| "kl": 0.00266265869140625, | |
| "learning_rate": 5.384615384615384e-07, | |
| "loss": 0.0001, | |
| "reward": 0.03125, | |
| "reward_std": 0.059198048897087574, | |
| "rewards/correctness_reward_func": 0.015625, | |
| "rewards/strict_format_reward_func": 0.015625, | |
| "step": 14 | |
| }, | |
| { | |
| "completion_length": 702.47265625, | |
| "epoch": 0.01741654571843251, | |
| "grad_norm": 0.3022926592468717, | |
| "kl": 0.00206756591796875, | |
| "learning_rate": 5.769230769230768e-07, | |
| "loss": 0.0001, | |
| "reward": 0.03125, | |
| "reward_std": 0.049292195588350296, | |
| "rewards/correctness_reward_func": 0.0234375, | |
| "rewards/strict_format_reward_func": 0.0078125, | |
| "step": 15 | |
| }, | |
| { | |
| "completion_length": 714.55859375, | |
| "epoch": 0.01857764876632801, | |
| "grad_norm": 0.67295700611606, | |
| "kl": 0.006526947021484375, | |
| "learning_rate": 6.153846153846154e-07, | |
| "loss": 0.0003, | |
| "reward": 0.0546875, | |
| "reward_std": 0.04213257320225239, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.0078125, | |
| "step": 16 | |
| }, | |
| { | |
| "completion_length": 728.94921875, | |
| "epoch": 0.019738751814223514, | |
| "grad_norm": 0.12683035812623522, | |
| "kl": 0.003253936767578125, | |
| "learning_rate": 6.538461538461538e-07, | |
| "loss": 0.0001, | |
| "reward": 0.041015625, | |
| "reward_std": 0.0688234455883503, | |
| "rewards/correctness_reward_func": 0.03125, | |
| "rewards/strict_format_reward_func": 0.009765625, | |
| "step": 17 | |
| }, | |
| { | |
| "completion_length": 752.4609375, | |
| "epoch": 0.020899854862119014, | |
| "grad_norm": 0.7572918382078203, | |
| "kl": 0.002063751220703125, | |
| "learning_rate": 6.923076923076922e-07, | |
| "loss": 0.0001, | |
| "reward": 0.01953125, | |
| "reward_std": 0.0390625, | |
| "rewards/correctness_reward_func": 0.015625, | |
| "rewards/strict_format_reward_func": 0.00390625, | |
| "step": 18 | |
| }, | |
| { | |
| "completion_length": 686.39453125, | |
| "epoch": 0.022060957910014514, | |
| "grad_norm": 0.366311042586646, | |
| "kl": 0.00254058837890625, | |
| "learning_rate": 7.307692307692307e-07, | |
| "loss": 0.0001, | |
| "reward": 0.037109375, | |
| "reward_std": 0.07421875, | |
| "rewards/correctness_reward_func": 0.03125, | |
| "rewards/strict_format_reward_func": 0.005859375, | |
| "step": 19 | |
| }, | |
| { | |
| "completion_length": 814.58203125, | |
| "epoch": 0.023222060957910014, | |
| "grad_norm": 0.39037265038987773, | |
| "kl": 0.0024509429931640625, | |
| "learning_rate": 7.692307692307693e-07, | |
| "loss": 0.0001, | |
| "reward": 0.056640625, | |
| "reward_std": 0.1000734455883503, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.009765625, | |
| "step": 20 | |
| }, | |
| { | |
| "completion_length": 790.58203125, | |
| "epoch": 0.024383164005805515, | |
| "grad_norm": 0.16449028843577287, | |
| "kl": 0.0017910003662109375, | |
| "learning_rate": 8.076923076923077e-07, | |
| "loss": 0.0001, | |
| "reward": 0.021484375, | |
| "reward_std": 0.04296875, | |
| "rewards/correctness_reward_func": 0.015625, | |
| "rewards/strict_format_reward_func": 0.005859375, | |
| "step": 21 | |
| }, | |
| { | |
| "completion_length": 720.109375, | |
| "epoch": 0.025544267053701015, | |
| "grad_norm": 0.16050190810020273, | |
| "kl": 0.000865936279296875, | |
| "learning_rate": 8.461538461538461e-07, | |
| "loss": 0.0, | |
| "reward": 0.041015625, | |
| "reward_std": 0.08203125, | |
| "rewards/correctness_reward_func": 0.03125, | |
| "rewards/strict_format_reward_func": 0.009765625, | |
| "step": 22 | |
| }, | |
| { | |
| "completion_length": 772.921875, | |
| "epoch": 0.026705370101596515, | |
| "grad_norm": 1.022693227541973, | |
| "kl": 0.010603904724121094, | |
| "learning_rate": 8.846153846153846e-07, | |
| "loss": 0.0004, | |
| "reward": 0.0234375, | |
| "reward_std": 0.046875, | |
| "rewards/correctness_reward_func": 0.015625, | |
| "rewards/strict_format_reward_func": 0.0078125, | |
| "step": 23 | |
| }, | |
| { | |
| "completion_length": 785.9296875, | |
| "epoch": 0.02786647314949202, | |
| "grad_norm": 0.4975098781878154, | |
| "kl": 0.0006268024444580078, | |
| "learning_rate": 9.230769230769231e-07, | |
| "loss": 0.0, | |
| "reward": 0.0546875, | |
| "reward_std": 0.10463257133960724, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.0078125, | |
| "step": 24 | |
| }, | |
| { | |
| "completion_length": 808.91796875, | |
| "epoch": 0.02902757619738752, | |
| "grad_norm": 0.07636536312105723, | |
| "kl": 0.0006952285766601562, | |
| "learning_rate": 9.615384615384615e-07, | |
| "loss": 0.0, | |
| "reward": 0.03515625, | |
| "reward_std": 0.055702777579426765, | |
| "rewards/correctness_reward_func": 0.0234375, | |
| "rewards/strict_format_reward_func": 0.01171875, | |
| "step": 25 | |
| }, | |
| { | |
| "completion_length": 677.65234375, | |
| "epoch": 0.03018867924528302, | |
| "grad_norm": 105.92855708885655, | |
| "kl": 0.048847198486328125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.002, | |
| "reward": 0.033203125, | |
| "reward_std": 0.06640625, | |
| "rewards/correctness_reward_func": 0.015625, | |
| "rewards/strict_format_reward_func": 0.017578125, | |
| "step": 26 | |
| }, | |
| { | |
| "completion_length": 657.85546875, | |
| "epoch": 0.03134978229317852, | |
| "grad_norm": 0.0861458003290139, | |
| "kl": 0.004261970520019531, | |
| "learning_rate": 9.999964611162971e-07, | |
| "loss": 0.0002, | |
| "reward": 0.009765625, | |
| "reward_std": 0.01953125, | |
| "rewards/correctness_reward_func": 0.0078125, | |
| "rewards/strict_format_reward_func": 0.001953125, | |
| "step": 27 | |
| }, | |
| { | |
| "completion_length": 726.2421875, | |
| "epoch": 0.03251088534107402, | |
| "grad_norm": 0.14527394620380304, | |
| "kl": 0.00072479248046875, | |
| "learning_rate": 9.999858445152839e-07, | |
| "loss": 0.0, | |
| "reward": 0.033203125, | |
| "reward_std": 0.053198445588350296, | |
| "rewards/correctness_reward_func": 0.03125, | |
| "rewards/strict_format_reward_func": 0.001953125, | |
| "step": 28 | |
| }, | |
| { | |
| "completion_length": 646.53125, | |
| "epoch": 0.03367198838896952, | |
| "grad_norm": 0.4038815325919152, | |
| "kl": 0.008038520812988281, | |
| "learning_rate": 9.999681503472433e-07, | |
| "loss": 0.0003, | |
| "reward": 0.02734375, | |
| "reward_std": 0.043573048897087574, | |
| "rewards/correctness_reward_func": 0.015625, | |
| "rewards/strict_format_reward_func": 0.01171875, | |
| "step": 29 | |
| }, | |
| { | |
| "completion_length": 676.234375, | |
| "epoch": 0.03483309143686502, | |
| "grad_norm": 24.530460542475996, | |
| "kl": 0.08559513092041016, | |
| "learning_rate": 9.99943378862646e-07, | |
| "loss": 0.0034, | |
| "reward": 0.076171875, | |
| "reward_std": 0.1358339935541153, | |
| "rewards/correctness_reward_func": 0.0625, | |
| "rewards/strict_format_reward_func": 0.013671875, | |
| "step": 30 | |
| }, | |
| { | |
| "completion_length": 669.62890625, | |
| "epoch": 0.035994194484760524, | |
| "grad_norm": 12.948432597412872, | |
| "kl": 0.0016450881958007812, | |
| "learning_rate": 9.999115304121457e-07, | |
| "loss": 0.0001, | |
| "reward": 0.033203125, | |
| "reward_std": 0.06640625, | |
| "rewards/correctness_reward_func": 0.0234375, | |
| "rewards/strict_format_reward_func": 0.009765625, | |
| "step": 31 | |
| }, | |
| { | |
| "completion_length": 632.80078125, | |
| "epoch": 0.03715529753265602, | |
| "grad_norm": 0.14738843577567481, | |
| "kl": 0.0011644363403320312, | |
| "learning_rate": 9.998726054465744e-07, | |
| "loss": 0.0, | |
| "reward": 0.0546875, | |
| "reward_std": 0.106073047965765, | |
| "rewards/correctness_reward_func": 0.0390625, | |
| "rewards/strict_format_reward_func": 0.015625, | |
| "step": 32 | |
| }, | |
| { | |
| "completion_length": 686.90234375, | |
| "epoch": 0.038316400580551524, | |
| "grad_norm": 6.995419155886516, | |
| "kl": 0.056095123291015625, | |
| "learning_rate": 9.998266045169354e-07, | |
| "loss": 0.0022, | |
| "reward": 0.052734375, | |
| "reward_std": 0.0922609455883503, | |
| "rewards/correctness_reward_func": 0.0390625, | |
| "rewards/strict_format_reward_func": 0.013671875, | |
| "step": 33 | |
| }, | |
| { | |
| "completion_length": 655.90234375, | |
| "epoch": 0.03947750362844703, | |
| "grad_norm": 3.5735767009613224, | |
| "kl": 0.013265609741210938, | |
| "learning_rate": 9.997735282743968e-07, | |
| "loss": 0.0005, | |
| "reward": 0.076171875, | |
| "reward_std": 0.13439351692795753, | |
| "rewards/correctness_reward_func": 0.0625, | |
| "rewards/strict_format_reward_func": 0.013671875, | |
| "step": 34 | |
| }, | |
| { | |
| "completion_length": 639.82421875, | |
| "epoch": 0.040638606676342524, | |
| "grad_norm": 1.8475352137430245, | |
| "kl": 0.0025959014892578125, | |
| "learning_rate": 9.997133774702812e-07, | |
| "loss": 0.0001, | |
| "reward": 0.05078125, | |
| "reward_std": 0.09682007133960724, | |
| "rewards/correctness_reward_func": 0.03125, | |
| "rewards/strict_format_reward_func": 0.01953125, | |
| "step": 35 | |
| }, | |
| { | |
| "completion_length": 715.3125, | |
| "epoch": 0.04179970972423803, | |
| "grad_norm": 13.632018237363075, | |
| "kl": 0.21777725219726562, | |
| "learning_rate": 9.996461529560552e-07, | |
| "loss": 0.0087, | |
| "reward": 0.0390625, | |
| "reward_std": 0.078125, | |
| "rewards/correctness_reward_func": 0.0234375, | |
| "rewards/strict_format_reward_func": 0.015625, | |
| "step": 36 | |
| }, | |
| { | |
| "completion_length": 631.8203125, | |
| "epoch": 0.042960812772133525, | |
| "grad_norm": 10.015816840984483, | |
| "kl": 0.00942230224609375, | |
| "learning_rate": 9.995718556833178e-07, | |
| "loss": 0.0004, | |
| "reward": 0.072265625, | |
| "reward_std": 0.141229297965765, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.025390625, | |
| "step": 37 | |
| }, | |
| { | |
| "completion_length": 636.9140625, | |
| "epoch": 0.04412191582002903, | |
| "grad_norm": 0.18443450708075712, | |
| "kl": 0.001720428466796875, | |
| "learning_rate": 9.994904867037865e-07, | |
| "loss": 0.0001, | |
| "reward": 0.033203125, | |
| "reward_std": 0.046594543382525444, | |
| "rewards/correctness_reward_func": 0.015625, | |
| "rewards/strict_format_reward_func": 0.017578125, | |
| "step": 38 | |
| }, | |
| { | |
| "completion_length": 667.6484375, | |
| "epoch": 0.045283018867924525, | |
| "grad_norm": 0.7286107675191874, | |
| "kl": 0.027004241943359375, | |
| "learning_rate": 9.994020471692832e-07, | |
| "loss": 0.0011, | |
| "reward": 0.0625, | |
| "reward_std": 0.10044586285948753, | |
| "rewards/correctness_reward_func": 0.0390625, | |
| "rewards/strict_format_reward_func": 0.0234375, | |
| "step": 39 | |
| }, | |
| { | |
| "completion_length": 695.68359375, | |
| "epoch": 0.04644412191582003, | |
| "grad_norm": 0.7697487269128109, | |
| "kl": 0.002681732177734375, | |
| "learning_rate": 9.993065383317162e-07, | |
| "loss": 0.0001, | |
| "reward": 0.078125, | |
| "reward_std": 0.12407248839735985, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.0234375, | |
| "step": 40 | |
| }, | |
| { | |
| "completion_length": 678.390625, | |
| "epoch": 0.04760522496371553, | |
| "grad_norm": 0.33385332935798095, | |
| "kl": 0.0028896331787109375, | |
| "learning_rate": 9.992039615430648e-07, | |
| "loss": 0.0001, | |
| "reward": 0.06640625, | |
| "reward_std": 0.1137621309608221, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.01953125, | |
| "step": 41 | |
| }, | |
| { | |
| "completion_length": 660.2421875, | |
| "epoch": 0.04876632801161103, | |
| "grad_norm": 5.2234598338711615, | |
| "kl": 0.03473663330078125, | |
| "learning_rate": 9.990943182553578e-07, | |
| "loss": 0.0014, | |
| "reward": 0.05859375, | |
| "reward_std": 0.11058359593153, | |
| "rewards/correctness_reward_func": 0.0234375, | |
| "rewards/strict_format_reward_func": 0.03515625, | |
| "step": 42 | |
| }, | |
| { | |
| "completion_length": 599.66015625, | |
| "epoch": 0.04992743105950653, | |
| "grad_norm": 0.3125631745813204, | |
| "kl": 0.0029144287109375, | |
| "learning_rate": 9.989776100206547e-07, | |
| "loss": 0.0001, | |
| "reward": 0.087890625, | |
| "reward_std": 0.1592714935541153, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.033203125, | |
| "step": 43 | |
| }, | |
| { | |
| "completion_length": 695.41015625, | |
| "epoch": 0.05108853410740203, | |
| "grad_norm": 0.31129440427202076, | |
| "kl": 0.002468109130859375, | |
| "learning_rate": 9.98853838491023e-07, | |
| "loss": 0.0001, | |
| "reward": 0.046875, | |
| "reward_std": 0.08240367192775011, | |
| "rewards/correctness_reward_func": 0.0234375, | |
| "rewards/strict_format_reward_func": 0.0234375, | |
| "step": 44 | |
| }, | |
| { | |
| "completion_length": 594.67578125, | |
| "epoch": 0.05224963715529753, | |
| "grad_norm": 0.2962376329942141, | |
| "kl": 0.00382232666015625, | |
| "learning_rate": 9.98723005418515e-07, | |
| "loss": 0.0002, | |
| "reward": 0.091796875, | |
| "reward_std": 0.1253724191337824, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.044921875, | |
| "step": 45 | |
| }, | |
| { | |
| "completion_length": 672.73046875, | |
| "epoch": 0.05341074020319303, | |
| "grad_norm": 3.778170631942383, | |
| "kl": 0.077880859375, | |
| "learning_rate": 9.985851126551428e-07, | |
| "loss": 0.0031, | |
| "reward": 0.103515625, | |
| "reward_std": 0.1781556848436594, | |
| "rewards/correctness_reward_func": 0.0625, | |
| "rewards/strict_format_reward_func": 0.041015625, | |
| "step": 46 | |
| }, | |
| { | |
| "completion_length": 603.15625, | |
| "epoch": 0.054571843251088534, | |
| "grad_norm": 0.3069672598383264, | |
| "kl": 0.006290435791015625, | |
| "learning_rate": 9.98440162152852e-07, | |
| "loss": 0.0003, | |
| "reward": 0.08984375, | |
| "reward_std": 0.1551719233393669, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.04296875, | |
| "step": 47 | |
| }, | |
| { | |
| "completion_length": 632.2890625, | |
| "epoch": 0.05573294629898404, | |
| "grad_norm": 1.2667105724676624, | |
| "kl": 0.029102325439453125, | |
| "learning_rate": 9.982881559634946e-07, | |
| "loss": 0.0012, | |
| "reward": 0.08203125, | |
| "reward_std": 0.14755274169147015, | |
| "rewards/correctness_reward_func": 0.03125, | |
| "rewards/strict_format_reward_func": 0.05078125, | |
| "step": 48 | |
| }, | |
| { | |
| "completion_length": 571.69921875, | |
| "epoch": 0.056894049346879534, | |
| "grad_norm": 51.00112062221978, | |
| "kl": 0.9501419067382812, | |
| "learning_rate": 9.981290962387997e-07, | |
| "loss": 0.0379, | |
| "reward": 0.07421875, | |
| "reward_std": 0.12532384134829044, | |
| "rewards/correctness_reward_func": 0.015625, | |
| "rewards/strict_format_reward_func": 0.05859375, | |
| "step": 49 | |
| }, | |
| { | |
| "completion_length": 595.40234375, | |
| "epoch": 0.05805515239477504, | |
| "grad_norm": 1.04581231978298, | |
| "kl": 0.01190185546875, | |
| "learning_rate": 9.979629852303425e-07, | |
| "loss": 0.0005, | |
| "reward": 0.11328125, | |
| "reward_std": 0.18154333159327507, | |
| "rewards/correctness_reward_func": 0.03125, | |
| "rewards/strict_format_reward_func": 0.08203125, | |
| "step": 50 | |
| }, | |
| { | |
| "completion_length": 561.47265625, | |
| "epoch": 0.059216255442670535, | |
| "grad_norm": 0.7109293653864751, | |
| "kl": 0.0103302001953125, | |
| "learning_rate": 9.977898252895134e-07, | |
| "loss": 0.0004, | |
| "reward": 0.13671875, | |
| "reward_std": 0.2260015867650509, | |
| "rewards/correctness_reward_func": 0.0390625, | |
| "rewards/strict_format_reward_func": 0.09765625, | |
| "step": 51 | |
| }, | |
| { | |
| "completion_length": 589.95703125, | |
| "epoch": 0.06037735849056604, | |
| "grad_norm": 2.9062465124154735, | |
| "kl": 0.00753021240234375, | |
| "learning_rate": 9.976096188674836e-07, | |
| "loss": 0.0003, | |
| "reward": 0.12890625, | |
| "reward_std": 0.19242635369300842, | |
| "rewards/correctness_reward_func": 0.0234375, | |
| "rewards/strict_format_reward_func": 0.10546875, | |
| "step": 52 | |
| }, | |
| { | |
| "completion_length": 537.0234375, | |
| "epoch": 0.06153846153846154, | |
| "grad_norm": 22.409980149466673, | |
| "kl": 0.2718505859375, | |
| "learning_rate": 9.974223685151718e-07, | |
| "loss": 0.0109, | |
| "reward": 0.12890625, | |
| "reward_std": 0.1836371347308159, | |
| "rewards/correctness_reward_func": 0.0234375, | |
| "rewards/strict_format_reward_func": 0.10546875, | |
| "step": 53 | |
| }, | |
| { | |
| "completion_length": 531.80859375, | |
| "epoch": 0.06269956458635705, | |
| "grad_norm": 2.8826743442454297, | |
| "kl": 0.122955322265625, | |
| "learning_rate": 9.972280768832067e-07, | |
| "loss": 0.0049, | |
| "reward": 0.220703125, | |
| "reward_std": 0.3008614853024483, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.126953125, | |
| "step": 54 | |
| }, | |
| { | |
| "completion_length": 611.5, | |
| "epoch": 0.06386066763425254, | |
| "grad_norm": 2.352779269743852, | |
| "kl": 0.02130889892578125, | |
| "learning_rate": 9.970267467218902e-07, | |
| "loss": 0.0009, | |
| "reward": 0.232421875, | |
| "reward_std": 0.2863368093967438, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.177734375, | |
| "step": 55 | |
| }, | |
| { | |
| "completion_length": 555.91796875, | |
| "epoch": 0.06502177068214804, | |
| "grad_norm": 0.9680034082980018, | |
| "kl": 0.01513671875, | |
| "learning_rate": 9.968183808811585e-07, | |
| "loss": 0.0006, | |
| "reward": 0.24609375, | |
| "reward_std": 0.3278784677386284, | |
| "rewards/correctness_reward_func": 0.0703125, | |
| "rewards/strict_format_reward_func": 0.17578125, | |
| "step": 56 | |
| }, | |
| { | |
| "completion_length": 561.3203125, | |
| "epoch": 0.06618287373004354, | |
| "grad_norm": 5.155662797346911, | |
| "kl": 0.1178741455078125, | |
| "learning_rate": 9.966029823105415e-07, | |
| "loss": 0.0047, | |
| "reward": 0.291015625, | |
| "reward_std": 0.3391275480389595, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.197265625, | |
| "step": 57 | |
| }, | |
| { | |
| "completion_length": 574.9453125, | |
| "epoch": 0.06734397677793905, | |
| "grad_norm": 3.5240176122453306, | |
| "kl": 0.0246734619140625, | |
| "learning_rate": 9.96380554059121e-07, | |
| "loss": 0.001, | |
| "reward": 0.291015625, | |
| "reward_std": 0.31832827627658844, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.212890625, | |
| "step": 58 | |
| }, | |
| { | |
| "completion_length": 583.93359375, | |
| "epoch": 0.06850507982583454, | |
| "grad_norm": 3.558262716521494, | |
| "kl": 0.0449981689453125, | |
| "learning_rate": 9.961510992754882e-07, | |
| "loss": 0.0018, | |
| "reward": 0.28515625, | |
| "reward_std": 0.26169369369745255, | |
| "rewards/correctness_reward_func": 0.0703125, | |
| "rewards/strict_format_reward_func": 0.21484375, | |
| "step": 59 | |
| }, | |
| { | |
| "completion_length": 498.65625, | |
| "epoch": 0.06966618287373004, | |
| "grad_norm": 2.811935806633887, | |
| "kl": 0.0194091796875, | |
| "learning_rate": 9.959146212076978e-07, | |
| "loss": 0.0008, | |
| "reward": 0.33984375, | |
| "reward_std": 0.314444862306118, | |
| "rewards/correctness_reward_func": 0.0625, | |
| "rewards/strict_format_reward_func": 0.27734375, | |
| "step": 60 | |
| }, | |
| { | |
| "completion_length": 478.53125, | |
| "epoch": 0.07082728592162554, | |
| "grad_norm": 4.984861468141312, | |
| "kl": 0.107757568359375, | |
| "learning_rate": 9.95671123203224e-07, | |
| "loss": 0.0043, | |
| "reward": 0.361328125, | |
| "reward_std": 0.33764002099633217, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.267578125, | |
| "step": 61 | |
| }, | |
| { | |
| "completion_length": 512.39453125, | |
| "epoch": 0.07198838896952105, | |
| "grad_norm": 1.007778929339388, | |
| "kl": 0.01971435546875, | |
| "learning_rate": 9.954206087089105e-07, | |
| "loss": 0.0008, | |
| "reward": 0.3203125, | |
| "reward_std": 0.2790137939155102, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.2734375, | |
| "step": 62 | |
| }, | |
| { | |
| "completion_length": 516.03515625, | |
| "epoch": 0.07314949201741655, | |
| "grad_norm": 0.5960945531101304, | |
| "kl": 0.0254974365234375, | |
| "learning_rate": 9.951630812709244e-07, | |
| "loss": 0.001, | |
| "reward": 0.373046875, | |
| "reward_std": 0.3191210627555847, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.279296875, | |
| "step": 63 | |
| }, | |
| { | |
| "completion_length": 478.65625, | |
| "epoch": 0.07431059506531204, | |
| "grad_norm": 3.3120058314917094, | |
| "kl": 0.027435302734375, | |
| "learning_rate": 9.948985445347044e-07, | |
| "loss": 0.0011, | |
| "reward": 0.396484375, | |
| "reward_std": 0.30096181109547615, | |
| "rewards/correctness_reward_func": 0.0703125, | |
| "rewards/strict_format_reward_func": 0.326171875, | |
| "step": 64 | |
| }, | |
| { | |
| "completion_length": 453.5390625, | |
| "epoch": 0.07547169811320754, | |
| "grad_norm": 10.565800054920043, | |
| "kl": 0.06463623046875, | |
| "learning_rate": 9.946270022449093e-07, | |
| "loss": 0.0026, | |
| "reward": 0.40234375, | |
| "reward_std": 0.30288316309452057, | |
| "rewards/correctness_reward_func": 0.0703125, | |
| "rewards/strict_format_reward_func": 0.33203125, | |
| "step": 65 | |
| }, | |
| { | |
| "completion_length": 467.4609375, | |
| "epoch": 0.07663280116110305, | |
| "grad_norm": 8.848702328863897, | |
| "kl": 0.026763916015625, | |
| "learning_rate": 9.94348458245365e-07, | |
| "loss": 0.0011, | |
| "reward": 0.484375, | |
| "reward_std": 0.3539407253265381, | |
| "rewards/correctness_reward_func": 0.1640625, | |
| "rewards/strict_format_reward_func": 0.3203125, | |
| "step": 66 | |
| }, | |
| { | |
| "completion_length": 460.51171875, | |
| "epoch": 0.07779390420899855, | |
| "grad_norm": 0.8276997968349737, | |
| "kl": 0.028564453125, | |
| "learning_rate": 9.940629164790118e-07, | |
| "loss": 0.0011, | |
| "reward": 0.390625, | |
| "reward_std": 0.2685689851641655, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.3359375, | |
| "step": 67 | |
| }, | |
| { | |
| "completion_length": 432.30078125, | |
| "epoch": 0.07895500725689406, | |
| "grad_norm": 1.6501741381150759, | |
| "kl": 0.068115234375, | |
| "learning_rate": 9.937703809878454e-07, | |
| "loss": 0.0027, | |
| "reward": 0.447265625, | |
| "reward_std": 0.32472314685583115, | |
| "rewards/correctness_reward_func": 0.109375, | |
| "rewards/strict_format_reward_func": 0.337890625, | |
| "step": 68 | |
| }, | |
| { | |
| "completion_length": 433.5625, | |
| "epoch": 0.08011611030478955, | |
| "grad_norm": 5.548040313902584, | |
| "kl": 0.0538330078125, | |
| "learning_rate": 9.934708559128622e-07, | |
| "loss": 0.0022, | |
| "reward": 0.48046875, | |
| "reward_std": 0.3404357209801674, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.36328125, | |
| "step": 69 | |
| }, | |
| { | |
| "completion_length": 411.9921875, | |
| "epoch": 0.08127721335268505, | |
| "grad_norm": 4.54760002369526, | |
| "kl": 0.0399169921875, | |
| "learning_rate": 9.931643454939998e-07, | |
| "loss": 0.0016, | |
| "reward": 0.505859375, | |
| "reward_std": 0.3411124534904957, | |
| "rewards/correctness_reward_func": 0.1484375, | |
| "rewards/strict_format_reward_func": 0.357421875, | |
| "step": 70 | |
| }, | |
| { | |
| "completion_length": 374.98828125, | |
| "epoch": 0.08243831640058055, | |
| "grad_norm": 0.7476462962662087, | |
| "kl": 0.06536865234375, | |
| "learning_rate": 9.928508540700772e-07, | |
| "loss": 0.0026, | |
| "reward": 0.458984375, | |
| "reward_std": 0.2434551902115345, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.365234375, | |
| "step": 71 | |
| }, | |
| { | |
| "completion_length": 371.80859375, | |
| "epoch": 0.08359941944847606, | |
| "grad_norm": 0.31697483250235303, | |
| "kl": 0.04022216796875, | |
| "learning_rate": 9.925303860787333e-07, | |
| "loss": 0.0016, | |
| "reward": 0.427734375, | |
| "reward_std": 0.24097520112991333, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.373046875, | |
| "step": 72 | |
| }, | |
| { | |
| "completion_length": 408.953125, | |
| "epoch": 0.08476052249637156, | |
| "grad_norm": 2.342541131973969, | |
| "kl": 0.0394287109375, | |
| "learning_rate": 9.92202946056364e-07, | |
| "loss": 0.0016, | |
| "reward": 0.447265625, | |
| "reward_std": 0.2658774182200432, | |
| "rewards/correctness_reward_func": 0.0625, | |
| "rewards/strict_format_reward_func": 0.384765625, | |
| "step": 73 | |
| }, | |
| { | |
| "completion_length": 367.03125, | |
| "epoch": 0.08592162554426705, | |
| "grad_norm": 0.457004333832563, | |
| "kl": 0.05194091796875, | |
| "learning_rate": 9.918685386380572e-07, | |
| "loss": 0.0021, | |
| "reward": 0.478515625, | |
| "reward_std": 0.22598736733198166, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.400390625, | |
| "step": 74 | |
| }, | |
| { | |
| "completion_length": 423.35546875, | |
| "epoch": 0.08708272859216255, | |
| "grad_norm": 81.83417853597544, | |
| "kl": 0.247314453125, | |
| "learning_rate": 9.915271685575296e-07, | |
| "loss": 0.0099, | |
| "reward": 0.48828125, | |
| "reward_std": 0.3054451234638691, | |
| "rewards/correctness_reward_func": 0.1015625, | |
| "rewards/strict_format_reward_func": 0.38671875, | |
| "step": 75 | |
| }, | |
| { | |
| "completion_length": 312.484375, | |
| "epoch": 0.08824383164005806, | |
| "grad_norm": 1.780578379981416, | |
| "kl": 0.0880126953125, | |
| "learning_rate": 9.911788406470568e-07, | |
| "loss": 0.0035, | |
| "reward": 0.548828125, | |
| "reward_std": 0.32934778556227684, | |
| "rewards/correctness_reward_func": 0.125, | |
| "rewards/strict_format_reward_func": 0.423828125, | |
| "step": 76 | |
| }, | |
| { | |
| "completion_length": 338.2421875, | |
| "epoch": 0.08940493468795356, | |
| "grad_norm": 0.6544200585420813, | |
| "kl": 0.0731201171875, | |
| "learning_rate": 9.908235598374068e-07, | |
| "loss": 0.0029, | |
| "reward": 0.45703125, | |
| "reward_std": 0.1958453767001629, | |
| "rewards/correctness_reward_func": 0.0390625, | |
| "rewards/strict_format_reward_func": 0.41796875, | |
| "step": 77 | |
| }, | |
| { | |
| "completion_length": 268.171875, | |
| "epoch": 0.09056603773584905, | |
| "grad_norm": 0.5879416696261314, | |
| "kl": 0.108642578125, | |
| "learning_rate": 9.904613311577695e-07, | |
| "loss": 0.0044, | |
| "reward": 0.556640625, | |
| "reward_std": 0.22046153992414474, | |
| "rewards/correctness_reward_func": 0.140625, | |
| "rewards/strict_format_reward_func": 0.416015625, | |
| "step": 78 | |
| }, | |
| { | |
| "completion_length": 231.109375, | |
| "epoch": 0.09172714078374455, | |
| "grad_norm": 0.8976017470783989, | |
| "kl": 0.18408203125, | |
| "learning_rate": 9.900921597356855e-07, | |
| "loss": 0.0074, | |
| "reward": 0.515625, | |
| "reward_std": 0.20618988201022148, | |
| "rewards/correctness_reward_func": 0.0859375, | |
| "rewards/strict_format_reward_func": 0.4296875, | |
| "step": 79 | |
| }, | |
| { | |
| "completion_length": 196.59765625, | |
| "epoch": 0.09288824383164006, | |
| "grad_norm": 17.77907159035059, | |
| "kl": 0.263916015625, | |
| "learning_rate": 9.897160507969735e-07, | |
| "loss": 0.0106, | |
| "reward": 0.47265625, | |
| "reward_std": 0.19209052622318268, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.41796875, | |
| "step": 80 | |
| }, | |
| { | |
| "completion_length": 110.4609375, | |
| "epoch": 0.09404934687953556, | |
| "grad_norm": 1.3276411102732406, | |
| "kl": 0.3662109375, | |
| "learning_rate": 9.893330096656573e-07, | |
| "loss": 0.0147, | |
| "reward": 0.578125, | |
| "reward_std": 0.27833693847060204, | |
| "rewards/correctness_reward_func": 0.140625, | |
| "rewards/strict_format_reward_func": 0.4375, | |
| "step": 81 | |
| }, | |
| { | |
| "completion_length": 77.88671875, | |
| "epoch": 0.09521044992743107, | |
| "grad_norm": 1.9379785150930775, | |
| "kl": 0.51953125, | |
| "learning_rate": 9.889430417638883e-07, | |
| "loss": 0.0208, | |
| "reward": 0.51171875, | |
| "reward_std": 0.1544804871082306, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.45703125, | |
| "step": 82 | |
| }, | |
| { | |
| "completion_length": 73.9375, | |
| "epoch": 0.09637155297532655, | |
| "grad_norm": 0.8473473359273695, | |
| "kl": 0.48486328125, | |
| "learning_rate": 9.885461526118713e-07, | |
| "loss": 0.0194, | |
| "reward": 0.50390625, | |
| "reward_std": 0.14182795584201813, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.45703125, | |
| "step": 83 | |
| }, | |
| { | |
| "completion_length": 56.2109375, | |
| "epoch": 0.09753265602322206, | |
| "grad_norm": 2.281085454498393, | |
| "kl": 0.49072265625, | |
| "learning_rate": 9.88142347827784e-07, | |
| "loss": 0.0196, | |
| "reward": 0.4765625, | |
| "reward_std": 0.12851098738610744, | |
| "rewards/correctness_reward_func": 0.0234375, | |
| "rewards/strict_format_reward_func": 0.453125, | |
| "step": 84 | |
| }, | |
| { | |
| "completion_length": 37.875, | |
| "epoch": 0.09869375907111756, | |
| "grad_norm": 0.5788019582854786, | |
| "kl": 0.5927734375, | |
| "learning_rate": 9.877316331276993e-07, | |
| "loss": 0.0237, | |
| "reward": 0.470703125, | |
| "reward_std": 0.055291797965765, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.470703125, | |
| "step": 85 | |
| }, | |
| { | |
| "completion_length": 33.50390625, | |
| "epoch": 0.09985486211901307, | |
| "grad_norm": 6.104259815482374, | |
| "kl": 0.5439453125, | |
| "learning_rate": 9.873140143255034e-07, | |
| "loss": 0.0217, | |
| "reward": 0.517578125, | |
| "reward_std": 0.07212539482861757, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.470703125, | |
| "step": 86 | |
| }, | |
| { | |
| "completion_length": 20.76171875, | |
| "epoch": 0.10101596516690857, | |
| "grad_norm": 0.507931311814518, | |
| "kl": 0.5947265625, | |
| "learning_rate": 9.86889497332814e-07, | |
| "loss": 0.0238, | |
| "reward": 0.486328125, | |
| "reward_std": 0.02734375, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.486328125, | |
| "step": 87 | |
| }, | |
| { | |
| "completion_length": 16.58203125, | |
| "epoch": 0.10217706821480406, | |
| "grad_norm": 0.45325431756004353, | |
| "kl": 0.6630859375, | |
| "learning_rate": 9.864580881588958e-07, | |
| "loss": 0.0265, | |
| "reward": 0.501953125, | |
| "reward_std": 0.024041798897087574, | |
| "rewards/correctness_reward_func": 0.0078125, | |
| "rewards/strict_format_reward_func": 0.494140625, | |
| "step": 88 | |
| }, | |
| { | |
| "completion_length": 16.12109375, | |
| "epoch": 0.10333817126269956, | |
| "grad_norm": 0.38014072113379394, | |
| "kl": 0.6513671875, | |
| "learning_rate": 9.860197929105767e-07, | |
| "loss": 0.0261, | |
| "reward": 0.49609375, | |
| "reward_std": 0.0078125, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.49609375, | |
| "step": 89 | |
| }, | |
| { | |
| "completion_length": 19.7265625, | |
| "epoch": 0.10449927431059507, | |
| "grad_norm": 3.246980620297076, | |
| "kl": 0.6064453125, | |
| "learning_rate": 9.8557461779216e-07, | |
| "loss": 0.0243, | |
| "reward": 0.48828125, | |
| "reward_std": 0.0234375, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.48828125, | |
| "step": 90 | |
| }, | |
| { | |
| "completion_length": 15.8359375, | |
| "epoch": 0.10566037735849057, | |
| "grad_norm": 1.125767741710113, | |
| "kl": 0.6162109375, | |
| "learning_rate": 9.85122569105338e-07, | |
| "loss": 0.0247, | |
| "reward": 0.49609375, | |
| "reward_std": 0.0078125, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.49609375, | |
| "step": 91 | |
| }, | |
| { | |
| "completion_length": 22.44921875, | |
| "epoch": 0.10682148040638606, | |
| "grad_norm": 0.44874987843781083, | |
| "kl": 0.6083984375, | |
| "learning_rate": 9.846636532491012e-07, | |
| "loss": 0.0243, | |
| "reward": 0.48828125, | |
| "reward_std": 0.0234375, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.48828125, | |
| "step": 92 | |
| }, | |
| { | |
| "completion_length": 16.21484375, | |
| "epoch": 0.10798258345428156, | |
| "grad_norm": 2.7244778647827363, | |
| "kl": 0.796875, | |
| "learning_rate": 9.841978767196493e-07, | |
| "loss": 0.0319, | |
| "reward": 0.494140625, | |
| "reward_std": 0.01171875, | |
| "rewards/correctness_reward_func": 0.0, | |
| "rewards/strict_format_reward_func": 0.494140625, | |
| "step": 93 | |
| }, | |
| { | |
| "completion_length": 18.08984375, | |
| "epoch": 0.10914368650217707, | |
| "grad_norm": 0.900717061581606, | |
| "kl": 0.5712890625, | |
| "learning_rate": 9.83725246110298e-07, | |
| "loss": 0.0229, | |
| "reward": 0.49609375, | |
| "reward_std": 0.0234375, | |
| "rewards/correctness_reward_func": 0.0078125, | |
| "rewards/strict_format_reward_func": 0.48828125, | |
| "step": 94 | |
| }, | |
| { | |
| "completion_length": 19.921875, | |
| "epoch": 0.11030478955007257, | |
| "grad_norm": 0.2922094758318377, | |
| "kl": 0.62109375, | |
| "learning_rate": 9.832457681113865e-07, | |
| "loss": 0.0249, | |
| "reward": 0.51953125, | |
| "reward_std": 0.0546875, | |
| "rewards/correctness_reward_func": 0.03125, | |
| "rewards/strict_format_reward_func": 0.48828125, | |
| "step": 95 | |
| }, | |
| { | |
| "completion_length": 22.48828125, | |
| "epoch": 0.11146589259796807, | |
| "grad_norm": 1.5719055280514225, | |
| "kl": 0.5966796875, | |
| "learning_rate": 9.827594495101822e-07, | |
| "loss": 0.0239, | |
| "reward": 0.53125, | |
| "reward_std": 0.049292195588350296, | |
| "rewards/correctness_reward_func": 0.0390625, | |
| "rewards/strict_format_reward_func": 0.4921875, | |
| "step": 96 | |
| }, | |
| { | |
| "completion_length": 21.078125, | |
| "epoch": 0.11262699564586356, | |
| "grad_norm": 1.875138112811585, | |
| "kl": 0.62890625, | |
| "learning_rate": 9.822662971907852e-07, | |
| "loss": 0.0252, | |
| "reward": 0.505859375, | |
| "reward_std": 0.047479298897087574, | |
| "rewards/correctness_reward_func": 0.015625, | |
| "rewards/strict_format_reward_func": 0.490234375, | |
| "step": 97 | |
| }, | |
| { | |
| "completion_length": 20.328125, | |
| "epoch": 0.11378809869375907, | |
| "grad_norm": 0.46440948422543626, | |
| "kl": 0.5703125, | |
| "learning_rate": 9.8176631813403e-07, | |
| "loss": 0.0228, | |
| "reward": 0.5078125, | |
| "reward_std": 0.015625, | |
| "rewards/correctness_reward_func": 0.0078125, | |
| "rewards/strict_format_reward_func": 0.5, | |
| "step": 98 | |
| }, | |
| { | |
| "completion_length": 17.6953125, | |
| "epoch": 0.11494920174165457, | |
| "grad_norm": 2.350666772323524, | |
| "kl": 0.6298828125, | |
| "learning_rate": 9.812595194173874e-07, | |
| "loss": 0.0252, | |
| "reward": 0.50390625, | |
| "reward_std": 0.0234375, | |
| "rewards/correctness_reward_func": 0.0078125, | |
| "rewards/strict_format_reward_func": 0.49609375, | |
| "step": 99 | |
| }, | |
| { | |
| "completion_length": 30.88671875, | |
| "epoch": 0.11611030478955008, | |
| "grad_norm": 0.6769899791454503, | |
| "kl": 0.537109375, | |
| "learning_rate": 9.807459082148648e-07, | |
| "loss": 0.0215, | |
| "reward": 0.56640625, | |
| "reward_std": 0.0804273895919323, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.48828125, | |
| "step": 100 | |
| }, | |
| { | |
| "completion_length": 38.8203125, | |
| "epoch": 0.11727140783744558, | |
| "grad_norm": 9.613286964921125, | |
| "kl": 0.53515625, | |
| "learning_rate": 9.802254917969032e-07, | |
| "loss": 0.0214, | |
| "reward": 0.5390625, | |
| "reward_std": 0.1328125, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 101 | |
| }, | |
| { | |
| "completion_length": 21.05078125, | |
| "epoch": 0.11843251088534107, | |
| "grad_norm": 0.383590636095333, | |
| "kl": 0.5419921875, | |
| "learning_rate": 9.796982775302754e-07, | |
| "loss": 0.0217, | |
| "reward": 0.52734375, | |
| "reward_std": 0.057104695588350296, | |
| "rewards/correctness_reward_func": 0.03125, | |
| "rewards/strict_format_reward_func": 0.49609375, | |
| "step": 102 | |
| }, | |
| { | |
| "completion_length": 26.5234375, | |
| "epoch": 0.11959361393323657, | |
| "grad_norm": 0.45036244814830856, | |
| "kl": 0.53759765625, | |
| "learning_rate": 9.79164272877981e-07, | |
| "loss": 0.0215, | |
| "reward": 0.53125, | |
| "reward_std": 0.090448047965765, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 103 | |
| }, | |
| { | |
| "completion_length": 23.54296875, | |
| "epoch": 0.12075471698113208, | |
| "grad_norm": 2.2388808963947713, | |
| "kl": 0.5615234375, | |
| "learning_rate": 9.786234853991418e-07, | |
| "loss": 0.0224, | |
| "reward": 0.56640625, | |
| "reward_std": 0.08606802299618721, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.48828125, | |
| "step": 104 | |
| }, | |
| { | |
| "completion_length": 53.6875, | |
| "epoch": 0.12191582002902758, | |
| "grad_norm": 1.3962986009964624, | |
| "kl": 0.515625, | |
| "learning_rate": 9.780759227488937e-07, | |
| "loss": 0.0207, | |
| "reward": 0.55859375, | |
| "reward_std": 0.1321367770433426, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 105 | |
| }, | |
| { | |
| "completion_length": 45.9140625, | |
| "epoch": 0.12307692307692308, | |
| "grad_norm": 0.6991087377799369, | |
| "kl": 0.5751953125, | |
| "learning_rate": 9.775215926782787e-07, | |
| "loss": 0.023, | |
| "reward": 0.5390625, | |
| "reward_std": 0.12271332554519176, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 106 | |
| }, | |
| { | |
| "completion_length": 41.71484375, | |
| "epoch": 0.12423802612481857, | |
| "grad_norm": 1.113559415990131, | |
| "kl": 0.501953125, | |
| "learning_rate": 9.769605030341355e-07, | |
| "loss": 0.0201, | |
| "reward": 0.494140625, | |
| "reward_std": 0.039666798897087574, | |
| "rewards/correctness_reward_func": 0.0078125, | |
| "rewards/strict_format_reward_func": 0.486328125, | |
| "step": 107 | |
| }, | |
| { | |
| "completion_length": 30.234375, | |
| "epoch": 0.1253991291727141, | |
| "grad_norm": 9.444282741370284, | |
| "kl": 0.52587890625, | |
| "learning_rate": 9.763926617589884e-07, | |
| "loss": 0.021, | |
| "reward": 0.529296875, | |
| "reward_std": 0.09918868914246559, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.482421875, | |
| "step": 108 | |
| }, | |
| { | |
| "completion_length": 36.328125, | |
| "epoch": 0.12656023222060958, | |
| "grad_norm": 0.5876435788826965, | |
| "kl": 0.49365234375, | |
| "learning_rate": 9.758180768909336e-07, | |
| "loss": 0.0198, | |
| "reward": 0.5625, | |
| "reward_std": 0.1117921955883503, | |
| "rewards/correctness_reward_func": 0.0703125, | |
| "rewards/strict_format_reward_func": 0.4921875, | |
| "step": 109 | |
| }, | |
| { | |
| "completion_length": 45.109375, | |
| "epoch": 0.12772133526850507, | |
| "grad_norm": 0.6967401340285807, | |
| "kl": 0.58203125, | |
| "learning_rate": 9.75236756563528e-07, | |
| "loss": 0.0233, | |
| "reward": 0.564453125, | |
| "reward_std": 0.1313234455883503, | |
| "rewards/correctness_reward_func": 0.0859375, | |
| "rewards/strict_format_reward_func": 0.478515625, | |
| "step": 110 | |
| }, | |
| { | |
| "completion_length": 22.265625, | |
| "epoch": 0.1288824383164006, | |
| "grad_norm": 4.157806679866693, | |
| "kl": 0.5859375, | |
| "learning_rate": 9.746487090056712e-07, | |
| "loss": 0.0235, | |
| "reward": 0.57421875, | |
| "reward_std": 0.1376468911767006, | |
| "rewards/correctness_reward_func": 0.0859375, | |
| "rewards/strict_format_reward_func": 0.48828125, | |
| "step": 111 | |
| }, | |
| { | |
| "completion_length": 43.33203125, | |
| "epoch": 0.13004354136429608, | |
| "grad_norm": 1.3608549571109825, | |
| "kl": 0.52099609375, | |
| "learning_rate": 9.740539425414911e-07, | |
| "loss": 0.0208, | |
| "reward": 0.548828125, | |
| "reward_std": 0.1078859455883503, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.494140625, | |
| "step": 112 | |
| }, | |
| { | |
| "completion_length": 27.4609375, | |
| "epoch": 0.1312046444121916, | |
| "grad_norm": 0.9303195344407332, | |
| "kl": 0.5302734375, | |
| "learning_rate": 9.734524655902252e-07, | |
| "loss": 0.0212, | |
| "reward": 0.5, | |
| "reward_std": 0.03125, | |
| "rewards/correctness_reward_func": 0.0078125, | |
| "rewards/strict_format_reward_func": 0.4921875, | |
| "step": 113 | |
| }, | |
| { | |
| "completion_length": 44.421875, | |
| "epoch": 0.13236574746008709, | |
| "grad_norm": 2.5989058419028512, | |
| "kl": 0.5224609375, | |
| "learning_rate": 9.728442866661012e-07, | |
| "loss": 0.0209, | |
| "reward": 0.5390625, | |
| "reward_std": 0.09286524448543787, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 114 | |
| }, | |
| { | |
| "completion_length": 55.43359375, | |
| "epoch": 0.13352685050798258, | |
| "grad_norm": 4.35900633763163, | |
| "kl": 0.513671875, | |
| "learning_rate": 9.722294143782171e-07, | |
| "loss": 0.0205, | |
| "reward": 0.5234375, | |
| "reward_std": 0.09476527757942677, | |
| "rewards/correctness_reward_func": 0.0390625, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 115 | |
| }, | |
| { | |
| "completion_length": 32.14453125, | |
| "epoch": 0.1346879535558781, | |
| "grad_norm": 2.5576115813189664, | |
| "kl": 0.48486328125, | |
| "learning_rate": 9.71607857430419e-07, | |
| "loss": 0.0194, | |
| "reward": 0.548828125, | |
| "reward_std": 0.1358339935541153, | |
| "rewards/correctness_reward_func": 0.0625, | |
| "rewards/strict_format_reward_func": 0.486328125, | |
| "step": 116 | |
| }, | |
| { | |
| "completion_length": 40.48828125, | |
| "epoch": 0.13584905660377358, | |
| "grad_norm": 5.3055382808430025, | |
| "kl": 0.55078125, | |
| "learning_rate": 9.709796246211776e-07, | |
| "loss": 0.022, | |
| "reward": 0.599609375, | |
| "reward_std": 0.1770055592060089, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.482421875, | |
| "step": 117 | |
| }, | |
| { | |
| "completion_length": 48.4296875, | |
| "epoch": 0.13701015965166907, | |
| "grad_norm": 1.9357783363747145, | |
| "kl": 0.52734375, | |
| "learning_rate": 9.70344724843465e-07, | |
| "loss": 0.0211, | |
| "reward": 0.623046875, | |
| "reward_std": 0.0844484455883503, | |
| "rewards/correctness_reward_func": 0.1328125, | |
| "rewards/strict_format_reward_func": 0.490234375, | |
| "step": 118 | |
| }, | |
| { | |
| "completion_length": 33.06640625, | |
| "epoch": 0.1381712626995646, | |
| "grad_norm": 0.9093609363691819, | |
| "kl": 0.48876953125, | |
| "learning_rate": 9.697031670846264e-07, | |
| "loss": 0.0195, | |
| "reward": 0.595703125, | |
| "reward_std": 0.1571781411767006, | |
| "rewards/correctness_reward_func": 0.109375, | |
| "rewards/strict_format_reward_func": 0.486328125, | |
| "step": 119 | |
| }, | |
| { | |
| "completion_length": 26.15234375, | |
| "epoch": 0.13933236574746008, | |
| "grad_norm": 1.2956378319464605, | |
| "kl": 0.52490234375, | |
| "learning_rate": 9.690549604262554e-07, | |
| "loss": 0.021, | |
| "reward": 0.611328125, | |
| "reward_std": 0.1670839935541153, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.494140625, | |
| "step": 120 | |
| }, | |
| { | |
| "completion_length": 23.31640625, | |
| "epoch": 0.1404934687953556, | |
| "grad_norm": 1.0404878413954166, | |
| "kl": 0.53173828125, | |
| "learning_rate": 9.684001140440638e-07, | |
| "loss": 0.0213, | |
| "reward": 0.533203125, | |
| "reward_std": 0.08553362637758255, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.486328125, | |
| "step": 121 | |
| }, | |
| { | |
| "completion_length": 33.63671875, | |
| "epoch": 0.1416545718432511, | |
| "grad_norm": 0.5491271196977439, | |
| "kl": 0.5263671875, | |
| "learning_rate": 9.677386372077523e-07, | |
| "loss": 0.0211, | |
| "reward": 0.537109375, | |
| "reward_std": 0.0798230916261673, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.490234375, | |
| "step": 122 | |
| }, | |
| { | |
| "completion_length": 27.7578125, | |
| "epoch": 0.14281567489114658, | |
| "grad_norm": 1.4086546997823266, | |
| "kl": 0.49462890625, | |
| "learning_rate": 9.670705392808795e-07, | |
| "loss": 0.0198, | |
| "reward": 0.642578125, | |
| "reward_std": 0.14176217839121819, | |
| "rewards/correctness_reward_func": 0.15625, | |
| "rewards/strict_format_reward_func": 0.486328125, | |
| "step": 123 | |
| }, | |
| { | |
| "completion_length": 46.31640625, | |
| "epoch": 0.1439767779390421, | |
| "grad_norm": 0.44935314029572543, | |
| "kl": 0.4541015625, | |
| "learning_rate": 9.663958297207285e-07, | |
| "loss": 0.0181, | |
| "reward": 0.5390625, | |
| "reward_std": 0.12601527571678162, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 124 | |
| }, | |
| { | |
| "completion_length": 42.33203125, | |
| "epoch": 0.14513788098693758, | |
| "grad_norm": 4.134753027574554, | |
| "kl": 0.45703125, | |
| "learning_rate": 9.657145180781742e-07, | |
| "loss": 0.0183, | |
| "reward": 0.513671875, | |
| "reward_std": 0.06089614052325487, | |
| "rewards/correctness_reward_func": 0.0234375, | |
| "rewards/strict_format_reward_func": 0.490234375, | |
| "step": 125 | |
| }, | |
| { | |
| "completion_length": 42.03515625, | |
| "epoch": 0.1462989840348331, | |
| "grad_norm": 0.4544547991850145, | |
| "kl": 0.40625, | |
| "learning_rate": 9.650266139975473e-07, | |
| "loss": 0.0163, | |
| "reward": 0.5703125, | |
| "reward_std": 0.12411524076014757, | |
| "rewards/correctness_reward_func": 0.0859375, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 126 | |
| }, | |
| { | |
| "completion_length": 31.16796875, | |
| "epoch": 0.1474600870827286, | |
| "grad_norm": 1.6260031691388108, | |
| "kl": 0.486328125, | |
| "learning_rate": 9.64332127216498e-07, | |
| "loss": 0.0194, | |
| "reward": 0.51171875, | |
| "reward_std": 0.0390625, | |
| "rewards/correctness_reward_func": 0.0234375, | |
| "rewards/strict_format_reward_func": 0.48828125, | |
| "step": 127 | |
| }, | |
| { | |
| "completion_length": 32.0, | |
| "epoch": 0.14862119013062408, | |
| "grad_norm": 0.5382173674869626, | |
| "kl": 0.4267578125, | |
| "learning_rate": 9.63631067565858e-07, | |
| "loss": 0.0171, | |
| "reward": 0.529296875, | |
| "reward_std": 0.053198445588350296, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.482421875, | |
| "step": 128 | |
| }, | |
| { | |
| "completion_length": 44.75390625, | |
| "epoch": 0.1497822931785196, | |
| "grad_norm": 0.7683808964372313, | |
| "kl": 0.4228515625, | |
| "learning_rate": 9.629234449695015e-07, | |
| "loss": 0.0169, | |
| "reward": 0.5625, | |
| "reward_std": 0.1478765867650509, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 129 | |
| }, | |
| { | |
| "completion_length": 40.52734375, | |
| "epoch": 0.1509433962264151, | |
| "grad_norm": 1.0117014376716735, | |
| "kl": 0.45263671875, | |
| "learning_rate": 9.62209269444205e-07, | |
| "loss": 0.0181, | |
| "reward": 0.65625, | |
| "reward_std": 0.218528650701046, | |
| "rewards/correctness_reward_func": 0.171875, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 130 | |
| }, | |
| { | |
| "completion_length": 28.6484375, | |
| "epoch": 0.1521044992743106, | |
| "grad_norm": 0.42513248914448737, | |
| "kl": 0.35986328125, | |
| "learning_rate": 9.614885510995047e-07, | |
| "loss": 0.0144, | |
| "reward": 0.5390625, | |
| "reward_std": 0.10979307442903519, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 131 | |
| }, | |
| { | |
| "completion_length": 48.9765625, | |
| "epoch": 0.1532656023222061, | |
| "grad_norm": 0.3690713785403208, | |
| "kl": 0.443359375, | |
| "learning_rate": 9.607613001375545e-07, | |
| "loss": 0.0177, | |
| "reward": 0.6015625, | |
| "reward_std": 0.1454593911767006, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 132 | |
| }, | |
| { | |
| "completion_length": 40.62890625, | |
| "epoch": 0.15442670537010159, | |
| "grad_norm": 0.3570306428369935, | |
| "kl": 0.40283203125, | |
| "learning_rate": 9.600275268529806e-07, | |
| "loss": 0.0161, | |
| "reward": 0.595703125, | |
| "reward_std": 0.0766359455883503, | |
| "rewards/correctness_reward_func": 0.1015625, | |
| "rewards/strict_format_reward_func": 0.494140625, | |
| "step": 133 | |
| }, | |
| { | |
| "completion_length": 48.47265625, | |
| "epoch": 0.1555878084179971, | |
| "grad_norm": 0.6699794243427841, | |
| "kl": 0.40283203125, | |
| "learning_rate": 9.592872416327364e-07, | |
| "loss": 0.0161, | |
| "reward": 0.537109375, | |
| "reward_std": 0.1280214972794056, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.482421875, | |
| "step": 134 | |
| }, | |
| { | |
| "completion_length": 43.74609375, | |
| "epoch": 0.1567489114658926, | |
| "grad_norm": 0.8744900024283108, | |
| "kl": 0.42919921875, | |
| "learning_rate": 9.58540454955955e-07, | |
| "loss": 0.0172, | |
| "reward": 0.568359375, | |
| "reward_std": 0.12251138500869274, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.474609375, | |
| "step": 135 | |
| }, | |
| { | |
| "completion_length": 35.98828125, | |
| "epoch": 0.1579100145137881, | |
| "grad_norm": 10.83989251533941, | |
| "kl": 0.43994140625, | |
| "learning_rate": 9.57787177393801e-07, | |
| "loss": 0.0176, | |
| "reward": 0.59765625, | |
| "reward_std": 0.1689911223948002, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 136 | |
| }, | |
| { | |
| "completion_length": 47.265625, | |
| "epoch": 0.1590711175616836, | |
| "grad_norm": 1.3945795045826723, | |
| "kl": 0.4638671875, | |
| "learning_rate": 9.570274196093216e-07, | |
| "loss": 0.0186, | |
| "reward": 0.5, | |
| "reward_std": 0.051385548897087574, | |
| "rewards/correctness_reward_func": 0.015625, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 137 | |
| }, | |
| { | |
| "completion_length": 37.30859375, | |
| "epoch": 0.1602322206095791, | |
| "grad_norm": 1.2085779669420207, | |
| "kl": 0.4755859375, | |
| "learning_rate": 9.562611923572943e-07, | |
| "loss": 0.019, | |
| "reward": 0.587890625, | |
| "reward_std": 0.16475876048207283, | |
| "rewards/correctness_reward_func": 0.109375, | |
| "rewards/strict_format_reward_func": 0.478515625, | |
| "step": 138 | |
| }, | |
| { | |
| "completion_length": 62.33203125, | |
| "epoch": 0.1613933236574746, | |
| "grad_norm": 0.6440211313776459, | |
| "kl": 0.419921875, | |
| "learning_rate": 9.554885064840755e-07, | |
| "loss": 0.0168, | |
| "reward": 0.51171875, | |
| "reward_std": 0.129510547965765, | |
| "rewards/correctness_reward_func": 0.0390625, | |
| "rewards/strict_format_reward_func": 0.47265625, | |
| "step": 139 | |
| }, | |
| { | |
| "completion_length": 42.2734375, | |
| "epoch": 0.1625544267053701, | |
| "grad_norm": 4.184412362449491, | |
| "kl": 0.458984375, | |
| "learning_rate": 9.547093729274473e-07, | |
| "loss": 0.0183, | |
| "reward": 0.56640625, | |
| "reward_std": 0.1319277435541153, | |
| "rewards/correctness_reward_func": 0.0859375, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 140 | |
| }, | |
| { | |
| "completion_length": 30.3203125, | |
| "epoch": 0.16371552975326562, | |
| "grad_norm": 0.9148362673963394, | |
| "kl": 0.4169921875, | |
| "learning_rate": 9.539238027164618e-07, | |
| "loss": 0.0167, | |
| "reward": 0.513671875, | |
| "reward_std": 0.053198445588350296, | |
| "rewards/correctness_reward_func": 0.0234375, | |
| "rewards/strict_format_reward_func": 0.490234375, | |
| "step": 141 | |
| }, | |
| { | |
| "completion_length": 26.02734375, | |
| "epoch": 0.1648766328011611, | |
| "grad_norm": 23.339183218639594, | |
| "kl": 0.4833984375, | |
| "learning_rate": 9.531318069712853e-07, | |
| "loss": 0.0193, | |
| "reward": 0.599609375, | |
| "reward_std": 0.15335902571678162, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.482421875, | |
| "step": 142 | |
| }, | |
| { | |
| "completion_length": 39.70703125, | |
| "epoch": 0.1660377358490566, | |
| "grad_norm": 4.918732336995348, | |
| "kl": 0.546875, | |
| "learning_rate": 9.523333969030412e-07, | |
| "loss": 0.0219, | |
| "reward": 0.623046875, | |
| "reward_std": 0.18786722049117088, | |
| "rewards/correctness_reward_func": 0.1328125, | |
| "rewards/strict_format_reward_func": 0.490234375, | |
| "step": 143 | |
| }, | |
| { | |
| "completion_length": 24.69140625, | |
| "epoch": 0.1671988388969521, | |
| "grad_norm": 2.3195586374061645, | |
| "kl": 0.46875, | |
| "learning_rate": 9.51528583813651e-07, | |
| "loss": 0.0187, | |
| "reward": 0.58203125, | |
| "reward_std": 0.13684775680303574, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.48828125, | |
| "step": 144 | |
| }, | |
| { | |
| "completion_length": 40.7109375, | |
| "epoch": 0.1683599419448476, | |
| "grad_norm": 0.6860447379166801, | |
| "kl": 0.49267578125, | |
| "learning_rate": 9.507173790956745e-07, | |
| "loss": 0.0197, | |
| "reward": 0.583984375, | |
| "reward_std": 0.1592714935541153, | |
| "rewards/correctness_reward_func": 0.1015625, | |
| "rewards/strict_format_reward_func": 0.482421875, | |
| "step": 145 | |
| }, | |
| { | |
| "completion_length": 60.76953125, | |
| "epoch": 0.16952104499274312, | |
| "grad_norm": 1.208065145658311, | |
| "kl": 0.55859375, | |
| "learning_rate": 9.498997942321483e-07, | |
| "loss": 0.0223, | |
| "reward": 0.5390625, | |
| "reward_std": 0.1552504375576973, | |
| "rewards/correctness_reward_func": 0.0703125, | |
| "rewards/strict_format_reward_func": 0.46875, | |
| "step": 146 | |
| }, | |
| { | |
| "completion_length": 34.5859375, | |
| "epoch": 0.1706821480406386, | |
| "grad_norm": 2.412402656297087, | |
| "kl": 0.5146484375, | |
| "learning_rate": 9.490758407964233e-07, | |
| "loss": 0.0206, | |
| "reward": 0.65234375, | |
| "reward_std": 0.14944713562726974, | |
| "rewards/correctness_reward_func": 0.171875, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 147 | |
| }, | |
| { | |
| "completion_length": 33.71875, | |
| "epoch": 0.1718432510885341, | |
| "grad_norm": 1.3484678762791356, | |
| "kl": 0.5146484375, | |
| "learning_rate": 9.482455304520012e-07, | |
| "loss": 0.0206, | |
| "reward": 0.50390625, | |
| "reward_std": 0.033667195588350296, | |
| "rewards/correctness_reward_func": 0.015625, | |
| "rewards/strict_format_reward_func": 0.48828125, | |
| "step": 148 | |
| }, | |
| { | |
| "completion_length": 43.68359375, | |
| "epoch": 0.17300435413642962, | |
| "grad_norm": 4.476292060280118, | |
| "kl": 0.5556640625, | |
| "learning_rate": 9.474088749523687e-07, | |
| "loss": 0.0222, | |
| "reward": 0.51953125, | |
| "reward_std": 0.051385547965765, | |
| "rewards/correctness_reward_func": 0.0390625, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 149 | |
| }, | |
| { | |
| "completion_length": 57.0078125, | |
| "epoch": 0.1741654571843251, | |
| "grad_norm": 3.243977251886513, | |
| "kl": 0.470703125, | |
| "learning_rate": 9.465658861408323e-07, | |
| "loss": 0.0188, | |
| "reward": 0.5703125, | |
| "reward_std": 0.1800113394856453, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 150 | |
| }, | |
| { | |
| "completion_length": 48.234375, | |
| "epoch": 0.1753265602322206, | |
| "grad_norm": 3.965397946904913, | |
| "kl": 0.51220703125, | |
| "learning_rate": 9.457165759503491e-07, | |
| "loss": 0.0204, | |
| "reward": 0.576171875, | |
| "reward_std": 0.14397033769637346, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.482421875, | |
| "step": 151 | |
| }, | |
| { | |
| "completion_length": 27.34765625, | |
| "epoch": 0.1764876632801161, | |
| "grad_norm": 5.956710202169321, | |
| "kl": 0.52734375, | |
| "learning_rate": 9.448609564033593e-07, | |
| "loss": 0.0211, | |
| "reward": 0.521484375, | |
| "reward_std": 0.13792734686285257, | |
| "rewards/correctness_reward_func": 0.046875, | |
| "rewards/strict_format_reward_func": 0.474609375, | |
| "step": 152 | |
| }, | |
| { | |
| "completion_length": 55.0234375, | |
| "epoch": 0.1776487663280116, | |
| "grad_norm": 1.9878758706657031, | |
| "kl": 0.625, | |
| "learning_rate": 9.439990396116148e-07, | |
| "loss": 0.025, | |
| "reward": 0.486328125, | |
| "reward_std": 0.1357191875576973, | |
| "rewards/correctness_reward_func": 0.03125, | |
| "rewards/strict_format_reward_func": 0.455078125, | |
| "step": 153 | |
| }, | |
| { | |
| "completion_length": 47.39453125, | |
| "epoch": 0.17880986937590712, | |
| "grad_norm": 0.3506677329292148, | |
| "kl": 0.568359375, | |
| "learning_rate": 9.431308377760093e-07, | |
| "loss": 0.0228, | |
| "reward": 0.587890625, | |
| "reward_std": 0.2043701931834221, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.470703125, | |
| "step": 154 | |
| }, | |
| { | |
| "completion_length": 34.31640625, | |
| "epoch": 0.1799709724238026, | |
| "grad_norm": 3.844767491669304, | |
| "kl": 0.51953125, | |
| "learning_rate": 9.422563631864037e-07, | |
| "loss": 0.0208, | |
| "reward": 0.513671875, | |
| "reward_std": 0.1123964935541153, | |
| "rewards/correctness_reward_func": 0.0390625, | |
| "rewards/strict_format_reward_func": 0.474609375, | |
| "step": 155 | |
| }, | |
| { | |
| "completion_length": 35.26171875, | |
| "epoch": 0.1811320754716981, | |
| "grad_norm": 10.543061352265745, | |
| "kl": 0.66748046875, | |
| "learning_rate": 9.413756282214537e-07, | |
| "loss": 0.0267, | |
| "reward": 0.560546875, | |
| "reward_std": 0.2216724045574665, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.466796875, | |
| "step": 156 | |
| }, | |
| { | |
| "completion_length": 53.375, | |
| "epoch": 0.18229317851959362, | |
| "grad_norm": 0.6406847195030275, | |
| "kl": 0.51416015625, | |
| "learning_rate": 9.404886453484335e-07, | |
| "loss": 0.0206, | |
| "reward": 0.486328125, | |
| "reward_std": 0.12331762351095676, | |
| "rewards/correctness_reward_func": 0.03125, | |
| "rewards/strict_format_reward_func": 0.455078125, | |
| "step": 157 | |
| }, | |
| { | |
| "completion_length": 33.31640625, | |
| "epoch": 0.1834542815674891, | |
| "grad_norm": 1.254136142861394, | |
| "kl": 0.6044921875, | |
| "learning_rate": 9.395954271230603e-07, | |
| "loss": 0.0242, | |
| "reward": 0.55078125, | |
| "reward_std": 0.1352296955883503, | |
| "rewards/correctness_reward_func": 0.0703125, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 158 | |
| }, | |
| { | |
| "completion_length": 38.45703125, | |
| "epoch": 0.18461538461538463, | |
| "grad_norm": 1.1951784005374884, | |
| "kl": 0.580078125, | |
| "learning_rate": 9.386959861893157e-07, | |
| "loss": 0.0232, | |
| "reward": 0.5, | |
| "reward_std": 0.09967818111181259, | |
| "rewards/correctness_reward_func": 0.03125, | |
| "rewards/strict_format_reward_func": 0.46875, | |
| "step": 159 | |
| }, | |
| { | |
| "completion_length": 49.9921875, | |
| "epoch": 0.18577648766328012, | |
| "grad_norm": 1.0186965999705826, | |
| "kl": 0.611328125, | |
| "learning_rate": 9.377903352792671e-07, | |
| "loss": 0.0244, | |
| "reward": 0.599609375, | |
| "reward_std": 0.19500285387039185, | |
| "rewards/correctness_reward_func": 0.1328125, | |
| "rewards/strict_format_reward_func": 0.466796875, | |
| "step": 160 | |
| }, | |
| { | |
| "completion_length": 25.6953125, | |
| "epoch": 0.1869375907111756, | |
| "grad_norm": 0.9553165570763839, | |
| "kl": 0.6337890625, | |
| "learning_rate": 9.368784872128876e-07, | |
| "loss": 0.0253, | |
| "reward": 0.65625, | |
| "reward_std": 0.23248206824064255, | |
| "rewards/correctness_reward_func": 0.1796875, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 161 | |
| }, | |
| { | |
| "completion_length": 29.96484375, | |
| "epoch": 0.18809869375907112, | |
| "grad_norm": 1.320529298833274, | |
| "kl": 0.677734375, | |
| "learning_rate": 9.359604548978741e-07, | |
| "loss": 0.0271, | |
| "reward": 0.546875, | |
| "reward_std": 0.1555742807686329, | |
| "rewards/correctness_reward_func": 0.0703125, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 162 | |
| }, | |
| { | |
| "completion_length": 37.44921875, | |
| "epoch": 0.1892597968069666, | |
| "grad_norm": 2.2124875651885754, | |
| "kl": 0.6181640625, | |
| "learning_rate": 9.35036251329465e-07, | |
| "loss": 0.0247, | |
| "reward": 0.513671875, | |
| "reward_std": 0.09588673710823059, | |
| "rewards/correctness_reward_func": 0.0390625, | |
| "rewards/strict_format_reward_func": 0.474609375, | |
| "step": 163 | |
| }, | |
| { | |
| "completion_length": 30.9609375, | |
| "epoch": 0.19042089985486213, | |
| "grad_norm": 0.7762326728671275, | |
| "kl": 0.755859375, | |
| "learning_rate": 9.341058895902562e-07, | |
| "loss": 0.0303, | |
| "reward": 0.615234375, | |
| "reward_std": 0.18324187397956848, | |
| "rewards/correctness_reward_func": 0.1328125, | |
| "rewards/strict_format_reward_func": 0.482421875, | |
| "step": 164 | |
| }, | |
| { | |
| "completion_length": 39.90234375, | |
| "epoch": 0.19158200290275762, | |
| "grad_norm": 1.5333968229845967, | |
| "kl": 0.681640625, | |
| "learning_rate": 9.331693828500158e-07, | |
| "loss": 0.0273, | |
| "reward": 0.53125, | |
| "reward_std": 0.11541798710823059, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 165 | |
| }, | |
| { | |
| "completion_length": 39.2265625, | |
| "epoch": 0.1927431059506531, | |
| "grad_norm": 3.341316216605523, | |
| "kl": 0.6484375, | |
| "learning_rate": 9.322267443654972e-07, | |
| "loss": 0.026, | |
| "reward": 0.552734375, | |
| "reward_std": 0.13462539482861757, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.474609375, | |
| "step": 166 | |
| }, | |
| { | |
| "completion_length": 41.33203125, | |
| "epoch": 0.19390420899854863, | |
| "grad_norm": 0.8334577080111663, | |
| "kl": 0.689453125, | |
| "learning_rate": 9.312779874802525e-07, | |
| "loss": 0.0276, | |
| "reward": 0.72265625, | |
| "reward_std": 0.2527032420039177, | |
| "rewards/correctness_reward_func": 0.2578125, | |
| "rewards/strict_format_reward_func": 0.46484375, | |
| "step": 167 | |
| }, | |
| { | |
| "completion_length": 27.4296875, | |
| "epoch": 0.19506531204644412, | |
| "grad_norm": 0.6225355591311358, | |
| "kl": 0.6796875, | |
| "learning_rate": 9.303231256244428e-07, | |
| "loss": 0.0272, | |
| "reward": 0.572265625, | |
| "reward_std": 0.1169070415198803, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.478515625, | |
| "step": 168 | |
| }, | |
| { | |
| "completion_length": 26.65625, | |
| "epoch": 0.19622641509433963, | |
| "grad_norm": 0.31164342343966356, | |
| "kl": 0.701171875, | |
| "learning_rate": 9.293621723146484e-07, | |
| "loss": 0.0281, | |
| "reward": 0.595703125, | |
| "reward_std": 0.1571781374514103, | |
| "rewards/correctness_reward_func": 0.109375, | |
| "rewards/strict_format_reward_func": 0.486328125, | |
| "step": 169 | |
| }, | |
| { | |
| "completion_length": 32.9609375, | |
| "epoch": 0.19738751814223512, | |
| "grad_norm": 0.8220338043159918, | |
| "kl": 0.6875, | |
| "learning_rate": 9.283951411536772e-07, | |
| "loss": 0.0274, | |
| "reward": 0.59375, | |
| "reward_std": 0.15472763776779175, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 170 | |
| }, | |
| { | |
| "completion_length": 29.8984375, | |
| "epoch": 0.19854862119013061, | |
| "grad_norm": 3.2547872245531146, | |
| "kl": 0.7255859375, | |
| "learning_rate": 9.274220458303726e-07, | |
| "loss": 0.029, | |
| "reward": 0.609375, | |
| "reward_std": 0.2199743092060089, | |
| "rewards/correctness_reward_func": 0.1328125, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 171 | |
| }, | |
| { | |
| "completion_length": 22.08203125, | |
| "epoch": 0.19970972423802613, | |
| "grad_norm": 1.0413843485165826, | |
| "kl": 0.7294921875, | |
| "learning_rate": 9.264429001194192e-07, | |
| "loss": 0.0292, | |
| "reward": 0.595703125, | |
| "reward_std": 0.2031683847308159, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.478515625, | |
| "step": 172 | |
| }, | |
| { | |
| "completion_length": 51.3671875, | |
| "epoch": 0.20087082728592162, | |
| "grad_norm": 3.778010417040405, | |
| "kl": 0.6826171875, | |
| "learning_rate": 9.254577178811482e-07, | |
| "loss": 0.0273, | |
| "reward": 0.6015625, | |
| "reward_std": 0.1914496347308159, | |
| "rewards/correctness_reward_func": 0.1328125, | |
| "rewards/strict_format_reward_func": 0.46875, | |
| "step": 173 | |
| }, | |
| { | |
| "completion_length": 44.84765625, | |
| "epoch": 0.20203193033381714, | |
| "grad_norm": 2.305304587495654, | |
| "kl": 0.7666015625, | |
| "learning_rate": 9.24466513061341e-07, | |
| "loss": 0.0307, | |
| "reward": 0.59375, | |
| "reward_std": 0.1900114268064499, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 174 | |
| }, | |
| { | |
| "completion_length": 36.86328125, | |
| "epoch": 0.20319303338171263, | |
| "grad_norm": 0.7728198355409298, | |
| "kl": 0.712890625, | |
| "learning_rate": 9.234692996910323e-07, | |
| "loss": 0.0285, | |
| "reward": 0.62890625, | |
| "reward_std": 0.1866152435541153, | |
| "rewards/correctness_reward_func": 0.1484375, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 175 | |
| }, | |
| { | |
| "completion_length": 29.98828125, | |
| "epoch": 0.20435413642960812, | |
| "grad_norm": 1.289972465399101, | |
| "kl": 0.6865234375, | |
| "learning_rate": 9.224660918863103e-07, | |
| "loss": 0.0275, | |
| "reward": 0.595703125, | |
| "reward_std": 0.13913594186306, | |
| "rewards/correctness_reward_func": 0.109375, | |
| "rewards/strict_format_reward_func": 0.486328125, | |
| "step": 176 | |
| }, | |
| { | |
| "completion_length": 34.23046875, | |
| "epoch": 0.20551523947750364, | |
| "grad_norm": 0.5793735600612212, | |
| "kl": 0.759765625, | |
| "learning_rate": 9.214569038481181e-07, | |
| "loss": 0.0304, | |
| "reward": 0.5625, | |
| "reward_std": 0.1687820851802826, | |
| "rewards/correctness_reward_func": 0.0859375, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 177 | |
| }, | |
| { | |
| "completion_length": 31.23828125, | |
| "epoch": 0.20667634252539913, | |
| "grad_norm": 0.5984526162665614, | |
| "kl": 0.771484375, | |
| "learning_rate": 9.20441749862052e-07, | |
| "loss": 0.0309, | |
| "reward": 0.68359375, | |
| "reward_std": 0.1762864589691162, | |
| "rewards/correctness_reward_func": 0.1953125, | |
| "rewards/strict_format_reward_func": 0.48828125, | |
| "step": 178 | |
| }, | |
| { | |
| "completion_length": 38.296875, | |
| "epoch": 0.20783744557329464, | |
| "grad_norm": 1.181839300217833, | |
| "kl": 0.7099609375, | |
| "learning_rate": 9.1942064429816e-07, | |
| "loss": 0.0284, | |
| "reward": 0.548828125, | |
| "reward_std": 0.10820978879928589, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.470703125, | |
| "step": 179 | |
| }, | |
| { | |
| "completion_length": 35.44921875, | |
| "epoch": 0.20899854862119013, | |
| "grad_norm": 7.06190367002311, | |
| "kl": 0.7001953125, | |
| "learning_rate": 9.18393601610737e-07, | |
| "loss": 0.028, | |
| "reward": 0.56640625, | |
| "reward_std": 0.1620996668934822, | |
| "rewards/correctness_reward_func": 0.0859375, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 180 | |
| }, | |
| { | |
| "completion_length": 24.76953125, | |
| "epoch": 0.21015965166908562, | |
| "grad_norm": 0.4282018680025159, | |
| "kl": 0.7275390625, | |
| "learning_rate": 9.173606363381218e-07, | |
| "loss": 0.0291, | |
| "reward": 0.654296875, | |
| "reward_std": 0.2599649503827095, | |
| "rewards/correctness_reward_func": 0.1640625, | |
| "rewards/strict_format_reward_func": 0.490234375, | |
| "step": 181 | |
| }, | |
| { | |
| "completion_length": 61.73046875, | |
| "epoch": 0.21132075471698114, | |
| "grad_norm": 0.666578405400966, | |
| "kl": 0.705078125, | |
| "learning_rate": 9.1632176310249e-07, | |
| "loss": 0.0283, | |
| "reward": 0.54296875, | |
| "reward_std": 0.1953125, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.46484375, | |
| "step": 182 | |
| }, | |
| { | |
| "completion_length": 41.6875, | |
| "epoch": 0.21248185776487663, | |
| "grad_norm": 2.4086312779043495, | |
| "kl": 0.6748046875, | |
| "learning_rate": 9.152769966096481e-07, | |
| "loss": 0.027, | |
| "reward": 0.65625, | |
| "reward_std": 0.2158661186695099, | |
| "rewards/correctness_reward_func": 0.1796875, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 183 | |
| }, | |
| { | |
| "completion_length": 47.5078125, | |
| "epoch": 0.21364296081277212, | |
| "grad_norm": 0.7460531004888707, | |
| "kl": 0.6865234375, | |
| "learning_rate": 9.142263516488244e-07, | |
| "loss": 0.0275, | |
| "reward": 0.53125, | |
| "reward_std": 0.1276262328028679, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 184 | |
| }, | |
| { | |
| "completion_length": 38.33203125, | |
| "epoch": 0.21480406386066764, | |
| "grad_norm": 1.677852493854183, | |
| "kl": 0.6611328125, | |
| "learning_rate": 9.131698430924605e-07, | |
| "loss": 0.0264, | |
| "reward": 0.6015625, | |
| "reward_std": 0.1689911223948002, | |
| "rewards/correctness_reward_func": 0.1171875, | |
| "rewards/strict_format_reward_func": 0.484375, | |
| "step": 185 | |
| }, | |
| { | |
| "completion_length": 31.5859375, | |
| "epoch": 0.21596516690856313, | |
| "grad_norm": 21.23887712764663, | |
| "kl": 0.689453125, | |
| "learning_rate": 9.121074858959996e-07, | |
| "loss": 0.0275, | |
| "reward": 0.564453125, | |
| "reward_std": 0.09907388500869274, | |
| "rewards/correctness_reward_func": 0.0859375, | |
| "rewards/strict_format_reward_func": 0.478515625, | |
| "step": 186 | |
| }, | |
| { | |
| "completion_length": 41.9453125, | |
| "epoch": 0.21712626995645865, | |
| "grad_norm": 0.6508903252265714, | |
| "kl": 0.6748046875, | |
| "learning_rate": 9.110392950976764e-07, | |
| "loss": 0.027, | |
| "reward": 0.5234375, | |
| "reward_std": 0.137323047965765, | |
| "rewards/correctness_reward_func": 0.0546875, | |
| "rewards/strict_format_reward_func": 0.46875, | |
| "step": 187 | |
| }, | |
| { | |
| "completion_length": 62.2109375, | |
| "epoch": 0.21828737300435413, | |
| "grad_norm": 1.1278006956369315, | |
| "kl": 0.673828125, | |
| "learning_rate": 9.099652858183027e-07, | |
| "loss": 0.027, | |
| "reward": 0.56640625, | |
| "reward_std": 0.1318129375576973, | |
| "rewards/correctness_reward_func": 0.1015625, | |
| "rewards/strict_format_reward_func": 0.46484375, | |
| "step": 188 | |
| }, | |
| { | |
| "completion_length": 46.51171875, | |
| "epoch": 0.21944847605224962, | |
| "grad_norm": 1.1019281044279183, | |
| "kl": 0.7548828125, | |
| "learning_rate": 9.088854732610542e-07, | |
| "loss": 0.0302, | |
| "reward": 0.572265625, | |
| "reward_std": 0.1169070415198803, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.478515625, | |
| "step": 189 | |
| }, | |
| { | |
| "completion_length": 35.03125, | |
| "epoch": 0.22060957910014514, | |
| "grad_norm": 14.962735092929739, | |
| "kl": 0.6845703125, | |
| "learning_rate": 9.077998727112552e-07, | |
| "loss": 0.0274, | |
| "reward": 0.564453125, | |
| "reward_std": 0.1369277872145176, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.470703125, | |
| "step": 190 | |
| }, | |
| { | |
| "completion_length": 55.27734375, | |
| "epoch": 0.22177068214804063, | |
| "grad_norm": 0.3504770063047999, | |
| "kl": 0.654296875, | |
| "learning_rate": 9.067084995361623e-07, | |
| "loss": 0.0262, | |
| "reward": 0.55078125, | |
| "reward_std": 0.1510637328028679, | |
| "rewards/correctness_reward_func": 0.078125, | |
| "rewards/strict_format_reward_func": 0.47265625, | |
| "step": 191 | |
| }, | |
| { | |
| "completion_length": 47.34375, | |
| "epoch": 0.22293178519593615, | |
| "grad_norm": 1.8159467800873224, | |
| "kl": 0.7333984375, | |
| "learning_rate": 9.05611369184746e-07, | |
| "loss": 0.0293, | |
| "reward": 0.57421875, | |
| "reward_std": 0.1376468911767006, | |
| "rewards/correctness_reward_func": 0.09375, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 192 | |
| }, | |
| { | |
| "completion_length": 49.0390625, | |
| "epoch": 0.22409288824383164, | |
| "grad_norm": 3.120448246962212, | |
| "kl": 0.69140625, | |
| "learning_rate": 9.045084971874737e-07, | |
| "loss": 0.0277, | |
| "reward": 0.609375, | |
| "reward_std": 0.1610843911767006, | |
| "rewards/correctness_reward_func": 0.1328125, | |
| "rewards/strict_format_reward_func": 0.4765625, | |
| "step": 193 | |
| }, | |
| { | |
| "completion_length": 33.7421875, | |
| "epoch": 0.22525399129172713, | |
| "grad_norm": 2.5542170019130244, | |
| "kl": 0.6787109375, | |
| "learning_rate": 9.03399899156088e-07, | |
| "loss": 0.0272, | |
| "reward": 0.58984375, | |
| "reward_std": 0.1015625, | |
| "rewards/correctness_reward_func": 0.1015625, | |
| "rewards/strict_format_reward_func": 0.48828125, | |
| "step": 194 | |
| }, | |
| { | |
| "completion_length": 54.81640625, | |
| "epoch": 0.22641509433962265, | |
| "grad_norm": 0.6406888848138048, | |
| "kl": 0.7216796875, | |
| "learning_rate": 9.022855907833871e-07, | |
| "loss": 0.0288, | |
| "reward": 0.5390625, | |
| "reward_std": 0.1084902435541153, | |
| "rewards/correctness_reward_func": 0.0703125, | |
| "rewards/strict_format_reward_func": 0.46875, | |
| "step": 195 | |
| }, | |
| { | |
| "completion_length": 33.41796875, | |
| "epoch": 0.22757619738751814, | |
| "grad_norm": 0.3997318900990298, | |
| "kl": 0.7177734375, | |
| "learning_rate": 9.011655878430018e-07, | |
| "loss": 0.0287, | |
| "reward": 0.587890625, | |
| "reward_std": 0.16475242003798485, | |
| "rewards/correctness_reward_func": 0.109375, | |
| "rewards/strict_format_reward_func": 0.478515625, | |
| "step": 196 | |
| }, | |
| { | |
| "completion_length": 62.3671875, | |
| "epoch": 0.22873730043541365, | |
| "grad_norm": 1.585750352753696, | |
| "kl": 0.71484375, | |
| "learning_rate": 9.000399061891728e-07, | |
| "loss": 0.0286, | |
| "reward": 0.630859375, | |
| "reward_std": 0.141229297965765, | |
| "rewards/correctness_reward_func": 0.1484375, | |
| "rewards/strict_format_reward_func": 0.482421875, | |
| "step": 197 | |
| }, | |
| { | |
| "completion_length": 38.67578125, | |
| "epoch": 0.22989840348330914, | |
| "grad_norm": 5.358536856283286, | |
| "kl": 0.6640625, | |
| "learning_rate": 8.98908561756526e-07, | |
| "loss": 0.0265, | |
| "reward": 0.517578125, | |
| "reward_std": 0.055291798897087574, | |
| "rewards/correctness_reward_func": 0.0390625, | |
| "rewards/strict_format_reward_func": 0.478515625, | |
| "step": 198 | |
| }, | |
| { | |
| "completion_length": 30.2109375, | |
| "epoch": 0.23105950653120463, | |
| "grad_norm": 0.39601285118928503, | |
| "kl": 0.720703125, | |
| "learning_rate": 8.977715705598467e-07, | |
| "loss": 0.0288, | |
| "reward": 0.54296875, | |
| "reward_std": 0.09165100753307343, | |
| "rewards/correctness_reward_func": 0.0625, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 199 | |
| }, | |
| { | |
| "completion_length": 29.21484375, | |
| "epoch": 0.23222060957910015, | |
| "grad_norm": 1.491626514094341, | |
| "kl": 0.73046875, | |
| "learning_rate": 8.966289486938536e-07, | |
| "loss": 0.0292, | |
| "reward": 0.63671875, | |
| "reward_std": 0.1542871668934822, | |
| "rewards/correctness_reward_func": 0.15625, | |
| "rewards/strict_format_reward_func": 0.48046875, | |
| "step": 200 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 861, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 100, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": false | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 4, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |