| { | |
| "best_global_step": null, | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.9936628643852978, | |
| "eval_steps": 100, | |
| "global_step": 196, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 544.7745819091797, | |
| "epoch": 0.010139416983523447, | |
| "grad_norm": 0.2070285975933075, | |
| "learning_rate": 5e-08, | |
| "loss": -0.1577, | |
| "num_tokens": 636118.0, | |
| "reward": -0.35065557062625885, | |
| "reward_std": 0.656378798186779, | |
| "rewards/cosine_scaled_reward": -0.35400376841425896, | |
| "rewards/format_reward": 0.003348214435391128, | |
| "step": 1 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 549.2812843322754, | |
| "epoch": 0.020278833967046894, | |
| "grad_norm": 0.2276979684829712, | |
| "learning_rate": 1e-07, | |
| "loss": -0.1565, | |
| "num_tokens": 1263330.0, | |
| "reward": -0.3044062051922083, | |
| "reward_std": 0.6904742792248726, | |
| "rewards/cosine_scaled_reward": -0.31221868470311165, | |
| "rewards/format_reward": 0.007812500349245965, | |
| "step": 2 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 588.2891006469727, | |
| "epoch": 0.030418250950570342, | |
| "grad_norm": 0.19815148413181305, | |
| "learning_rate": 1.5e-07, | |
| "loss": -0.1123, | |
| "num_tokens": 1930557.0, | |
| "reward": -0.2540861386805773, | |
| "reward_std": 0.6807773411273956, | |
| "rewards/cosine_scaled_reward": -0.2618986228480935, | |
| "rewards/format_reward": 0.007812500232830644, | |
| "step": 3 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 552.0669898986816, | |
| "epoch": 0.04055766793409379, | |
| "grad_norm": 0.20235052704811096, | |
| "learning_rate": 2e-07, | |
| "loss": -0.1652, | |
| "num_tokens": 2562161.0, | |
| "reward": -0.3022355027496815, | |
| "reward_std": 0.6755559742450714, | |
| "rewards/cosine_scaled_reward": -0.3122801296412945, | |
| "rewards/format_reward": 0.010044643189758062, | |
| "step": 4 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 571.6172142028809, | |
| "epoch": 0.050697084917617236, | |
| "grad_norm": 0.19000709056854248, | |
| "learning_rate": 2.5e-07, | |
| "loss": -0.1267, | |
| "num_tokens": 3201114.0, | |
| "reward": -0.2917697671800852, | |
| "reward_std": 0.6964326798915863, | |
| "rewards/cosine_scaled_reward": -0.2984661813825369, | |
| "rewards/format_reward": 0.006696428870782256, | |
| "step": 5 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 561.6384201049805, | |
| "epoch": 0.060836501901140684, | |
| "grad_norm": 0.21064706146717072, | |
| "learning_rate": 3e-07, | |
| "loss": -0.1771, | |
| "num_tokens": 3833534.0, | |
| "reward": -0.28072800021618605, | |
| "reward_std": 0.6883805021643639, | |
| "rewards/cosine_scaled_reward": -0.2840762068517506, | |
| "rewards/format_reward": 0.003348214435391128, | |
| "step": 6 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 542.1149749755859, | |
| "epoch": 0.07097591888466413, | |
| "grad_norm": 0.22860734164714813, | |
| "learning_rate": 3.5e-07, | |
| "loss": -0.131, | |
| "num_tokens": 4443693.0, | |
| "reward": -0.23171719023957849, | |
| "reward_std": 0.7431022375822067, | |
| "rewards/cosine_scaled_reward": -0.24287789640948176, | |
| "rewards/format_reward": 0.011160714668221772, | |
| "step": 7 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 563.7890892028809, | |
| "epoch": 0.08111533586818757, | |
| "grad_norm": 0.19847826659679413, | |
| "learning_rate": 4e-07, | |
| "loss": -0.142, | |
| "num_tokens": 5076400.0, | |
| "reward": -0.3053848221898079, | |
| "reward_std": 0.6786770969629288, | |
| "rewards/cosine_scaled_reward": -0.3109651692211628, | |
| "rewards/format_reward": 0.005580357392318547, | |
| "step": 8 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 585.7589569091797, | |
| "epoch": 0.09125475285171103, | |
| "grad_norm": 0.1838308870792389, | |
| "learning_rate": 4.5e-07, | |
| "loss": -0.1486, | |
| "num_tokens": 5724568.0, | |
| "reward": -0.26803811825811863, | |
| "reward_std": 0.6804082244634628, | |
| "rewards/cosine_scaled_reward": -0.26915417425334454, | |
| "rewards/format_reward": 0.0011160714784637094, | |
| "step": 9 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 516.0636291503906, | |
| "epoch": 0.10139416983523447, | |
| "grad_norm": 0.6098268032073975, | |
| "learning_rate": 5e-07, | |
| "loss": -0.1271, | |
| "num_tokens": 6313689.0, | |
| "reward": -0.2813316825777292, | |
| "reward_std": 0.7330739945173264, | |
| "rewards/cosine_scaled_reward": -0.2880280986428261, | |
| "rewards/format_reward": 0.006696428870782256, | |
| "step": 10 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 568.8984718322754, | |
| "epoch": 0.11153358681875793, | |
| "grad_norm": 0.5140683054924011, | |
| "learning_rate": 5.5e-07, | |
| "loss": -0.1441, | |
| "num_tokens": 6960958.0, | |
| "reward": -0.2723624687641859, | |
| "reward_std": 0.6729818060994148, | |
| "rewards/cosine_scaled_reward": -0.275710666552186, | |
| "rewards/format_reward": 0.003348214435391128, | |
| "step": 11 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 584.2366409301758, | |
| "epoch": 0.12167300380228137, | |
| "grad_norm": 0.20556923747062683, | |
| "learning_rate": 6e-07, | |
| "loss": -0.1173, | |
| "num_tokens": 7615746.0, | |
| "reward": -0.21572664007544518, | |
| "reward_std": 0.7041841298341751, | |
| "rewards/cosine_scaled_reward": -0.2201909152790904, | |
| "rewards/format_reward": 0.004464285913854837, | |
| "step": 12 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 598.1797065734863, | |
| "epoch": 0.13181242078580482, | |
| "grad_norm": 0.19161486625671387, | |
| "learning_rate": 6.5e-07, | |
| "loss": -0.1216, | |
| "num_tokens": 8279899.0, | |
| "reward": -0.23263627663254738, | |
| "reward_std": 0.7194378077983856, | |
| "rewards/cosine_scaled_reward": -0.23933268897235394, | |
| "rewards/format_reward": 0.006696428870782256, | |
| "step": 13 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 539.4096183776855, | |
| "epoch": 0.14195183776932827, | |
| "grad_norm": 0.20079070329666138, | |
| "learning_rate": 7e-07, | |
| "loss": -0.1091, | |
| "num_tokens": 8895386.0, | |
| "reward": -0.21676091896370053, | |
| "reward_std": 0.7302467301487923, | |
| "rewards/cosine_scaled_reward": -0.22209045942872763, | |
| "rewards/format_reward": 0.0011160714784637094, | |
| "step": 14 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 581.7645378112793, | |
| "epoch": 0.1520912547528517, | |
| "grad_norm": 0.21947209537029266, | |
| "learning_rate": 7.5e-07, | |
| "loss": -0.097, | |
| "num_tokens": 9545087.0, | |
| "reward": -0.10511145042255521, | |
| "reward_std": 0.7483287081122398, | |
| "rewards/cosine_scaled_reward": -0.10957573121413589, | |
| "rewards/format_reward": 0.004464285797439516, | |
| "step": 15 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 587.3013725280762, | |
| "epoch": 0.16223067173637515, | |
| "grad_norm": 0.20107129216194153, | |
| "learning_rate": 8e-07, | |
| "loss": -0.0894, | |
| "num_tokens": 10198237.0, | |
| "reward": -0.14020115474704653, | |
| "reward_std": 0.7795401588082314, | |
| "rewards/cosine_scaled_reward": -0.14689757686574012, | |
| "rewards/format_reward": 0.006696428870782256, | |
| "step": 16 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 616.6361923217773, | |
| "epoch": 0.17237008871989862, | |
| "grad_norm": 0.17595961689949036, | |
| "learning_rate": 8.499999999999999e-07, | |
| "loss": -0.1258, | |
| "num_tokens": 10885255.0, | |
| "reward": -0.16032503126189113, | |
| "reward_std": 0.7151542007923126, | |
| "rewards/cosine_scaled_reward": -0.16478930541779846, | |
| "rewards/format_reward": 0.004464285913854837, | |
| "step": 17 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 614.3348388671875, | |
| "epoch": 0.18250950570342206, | |
| "grad_norm": 0.17326559126377106, | |
| "learning_rate": 9e-07, | |
| "loss": -0.1223, | |
| "num_tokens": 11559043.0, | |
| "reward": -0.05317553365603089, | |
| "reward_std": 0.8043533340096474, | |
| "rewards/cosine_scaled_reward": -0.055407675448805094, | |
| "rewards/format_reward": 0.0022321429569274187, | |
| "step": 18 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 576.6774749755859, | |
| "epoch": 0.1926489226869455, | |
| "grad_norm": 0.19331927597522736, | |
| "learning_rate": 9.499999999999999e-07, | |
| "loss": -0.0965, | |
| "num_tokens": 12207730.0, | |
| "reward": -0.01400054944679141, | |
| "reward_std": 0.8140848129987717, | |
| "rewards/cosine_scaled_reward": -0.017348763067275286, | |
| "rewards/format_reward": 0.003348214435391128, | |
| "step": 19 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 659.9911117553711, | |
| "epoch": 0.20278833967046894, | |
| "grad_norm": 0.16300565004348755, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0491, | |
| "num_tokens": 12919506.0, | |
| "reward": 0.12035302398726344, | |
| "reward_std": 0.7825085148215294, | |
| "rewards/cosine_scaled_reward": 0.11588872922584414, | |
| "rewards/format_reward": 0.004464285797439516, | |
| "step": 20 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 641.3013687133789, | |
| "epoch": 0.21292775665399238, | |
| "grad_norm": 0.24253493547439575, | |
| "learning_rate": 9.999203468625015e-07, | |
| "loss": -0.0523, | |
| "num_tokens": 13615384.0, | |
| "reward": 0.19580535404384136, | |
| "reward_std": 0.8355192169547081, | |
| "rewards/cosine_scaled_reward": 0.19357320480048656, | |
| "rewards/format_reward": 0.0022321429569274187, | |
| "step": 21 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 694.412971496582, | |
| "epoch": 0.22306717363751585, | |
| "grad_norm": 0.22512537240982056, | |
| "learning_rate": 9.99681412828496e-07, | |
| "loss": -0.0494, | |
| "num_tokens": 14370738.0, | |
| "reward": 0.2461150388699025, | |
| "reward_std": 0.7426658719778061, | |
| "rewards/cosine_scaled_reward": 0.24388288520276546, | |
| "rewards/format_reward": 0.0022321429569274187, | |
| "step": 22 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 675.4542617797852, | |
| "epoch": 0.2332065906210393, | |
| "grad_norm": 0.16492308676242828, | |
| "learning_rate": 9.992832740253644e-07, | |
| "loss": -0.0398, | |
| "num_tokens": 15114193.0, | |
| "reward": 0.2721172561869025, | |
| "reward_std": 0.8025888651609421, | |
| "rewards/cosine_scaled_reward": 0.2710011739982292, | |
| "rewards/format_reward": 0.0011160714784637094, | |
| "step": 23 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 719.147346496582, | |
| "epoch": 0.24334600760456274, | |
| "grad_norm": 0.1536169797182083, | |
| "learning_rate": 9.987260573051267e-07, | |
| "loss": -0.0321, | |
| "num_tokens": 15888093.0, | |
| "reward": 0.30394730158150196, | |
| "reward_std": 0.7235733941197395, | |
| "rewards/cosine_scaled_reward": 0.30394728668034077, | |
| "rewards/format_reward": 0.0, | |
| "step": 24 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 665.2589645385742, | |
| "epoch": 0.2534854245880862, | |
| "grad_norm": 0.15167948603630066, | |
| "learning_rate": 9.98009940204023e-07, | |
| "loss": -0.0196, | |
| "num_tokens": 16615493.0, | |
| "reward": 0.4243004210293293, | |
| "reward_std": 0.784424401819706, | |
| "rewards/cosine_scaled_reward": 0.42318433709442616, | |
| "rewards/format_reward": 0.0011160714784637094, | |
| "step": 25 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 680.0814971923828, | |
| "epoch": 0.26362484157160965, | |
| "grad_norm": 0.1787332445383072, | |
| "learning_rate": 9.971351508859486e-07, | |
| "loss": -0.0554, | |
| "num_tokens": 17343678.0, | |
| "reward": 0.5126287564635277, | |
| "reward_std": 0.7500302940607071, | |
| "rewards/cosine_scaled_reward": 0.5126287154853344, | |
| "rewards/format_reward": 0.0, | |
| "step": 26 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 691.2957992553711, | |
| "epoch": 0.2737642585551331, | |
| "grad_norm": 0.15422694385051727, | |
| "learning_rate": 9.961019680697591e-07, | |
| "loss": 0.0019, | |
| "num_tokens": 18091255.0, | |
| "reward": 0.478223854675889, | |
| "reward_std": 0.7884277924895287, | |
| "rewards/cosine_scaled_reward": 0.4771077586337924, | |
| "rewards/format_reward": 0.0011160714784637094, | |
| "step": 27 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 724.2433319091797, | |
| "epoch": 0.28390367553865653, | |
| "grad_norm": 0.14453695714473724, | |
| "learning_rate": 9.949107209404663e-07, | |
| "loss": 0.0015, | |
| "num_tokens": 18873817.0, | |
| "reward": 0.6080499514937401, | |
| "reward_std": 0.7710720151662827, | |
| "rewards/cosine_scaled_reward": 0.6080499142408371, | |
| "rewards/format_reward": 0.0, | |
| "step": 28 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 719.5837326049805, | |
| "epoch": 0.29404309252217997, | |
| "grad_norm": 0.14530107378959656, | |
| "learning_rate": 9.935617890443554e-07, | |
| "loss": 0.0026, | |
| "num_tokens": 19659300.0, | |
| "reward": 0.6268112398684025, | |
| "reward_std": 0.6641118600964546, | |
| "rewards/cosine_scaled_reward": 0.6268112063407898, | |
| "rewards/format_reward": 0.0, | |
| "step": 29 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 793.0312805175781, | |
| "epoch": 0.3041825095057034, | |
| "grad_norm": 0.1358717679977417, | |
| "learning_rate": 9.92055602168058e-07, | |
| "loss": -0.0059, | |
| "num_tokens": 20515344.0, | |
| "reward": 0.6840809024870396, | |
| "reward_std": 0.6523778513073921, | |
| "rewards/cosine_scaled_reward": 0.6840808726847172, | |
| "rewards/format_reward": 0.0, | |
| "step": 30 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 766.2176818847656, | |
| "epoch": 0.31432192648922685, | |
| "grad_norm": 0.13609299063682556, | |
| "learning_rate": 9.90392640201615e-07, | |
| "loss": 0.0048, | |
| "num_tokens": 21329483.0, | |
| "reward": 0.7440241314470768, | |
| "reward_std": 0.6845037117600441, | |
| "rewards/cosine_scaled_reward": 0.7440241128206253, | |
| "rewards/format_reward": 0.0, | |
| "step": 31 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 760.3225784301758, | |
| "epoch": 0.3244613434727503, | |
| "grad_norm": 0.1422865241765976, | |
| "learning_rate": 9.885734329855797e-07, | |
| "loss": 0.0076, | |
| "num_tokens": 22130476.0, | |
| "reward": 0.8551607951521873, | |
| "reward_std": 0.6532174497842789, | |
| "rewards/cosine_scaled_reward": 0.8551607728004456, | |
| "rewards/format_reward": 0.0, | |
| "step": 32 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 788.7120895385742, | |
| "epoch": 0.33460076045627374, | |
| "grad_norm": 0.13245096802711487, | |
| "learning_rate": 9.865985601422017e-07, | |
| "loss": -0.0077, | |
| "num_tokens": 22971218.0, | |
| "reward": 0.7257559858262539, | |
| "reward_std": 0.6099549047648907, | |
| "rewards/cosine_scaled_reward": 0.7302779853343964, | |
| "rewards/format_reward": 0.0, | |
| "step": 33 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 802.2344055175781, | |
| "epoch": 0.34474017743979724, | |
| "grad_norm": 0.13606438040733337, | |
| "learning_rate": 9.844686508907537e-07, | |
| "loss": -0.0022, | |
| "num_tokens": 23817508.0, | |
| "reward": 0.9126424044370651, | |
| "reward_std": 0.6410668790340424, | |
| "rewards/cosine_scaled_reward": 0.9126423671841621, | |
| "rewards/format_reward": 0.0, | |
| "step": 34 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 779.1975784301758, | |
| "epoch": 0.3548795944233207, | |
| "grad_norm": 0.13803985714912415, | |
| "learning_rate": 9.821843838470534e-07, | |
| "loss": -0.0014, | |
| "num_tokens": 24658397.0, | |
| "reward": 0.951663188636303, | |
| "reward_std": 0.6128395721316338, | |
| "rewards/cosine_scaled_reward": 0.9516631364822388, | |
| "rewards/format_reward": 0.0, | |
| "step": 35 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 790.4141006469727, | |
| "epoch": 0.3650190114068441, | |
| "grad_norm": 0.15671393275260925, | |
| "learning_rate": 9.797464868072486e-07, | |
| "loss": 0.0435, | |
| "num_tokens": 25494720.0, | |
| "reward": 0.952189639210701, | |
| "reward_std": 0.6134774163365364, | |
| "rewards/cosine_scaled_reward": 0.9521896243095398, | |
| "rewards/format_reward": 0.0, | |
| "step": 36 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 780.6808395385742, | |
| "epoch": 0.37515842839036756, | |
| "grad_norm": 0.13327902555465698, | |
| "learning_rate": 9.771557365159319e-07, | |
| "loss": -0.0051, | |
| "num_tokens": 26319738.0, | |
| "reward": 0.9673999920487404, | |
| "reward_std": 0.5673549436032772, | |
| "rewards/cosine_scaled_reward": 0.9673999547958374, | |
| "rewards/format_reward": 0.0, | |
| "step": 37 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 797.1105194091797, | |
| "epoch": 0.385297845373891, | |
| "grad_norm": 0.14108316600322723, | |
| "learning_rate": 9.744129584186597e-07, | |
| "loss": -0.0108, | |
| "num_tokens": 27160557.0, | |
| "reward": 0.9532012119889259, | |
| "reward_std": 0.5637913830578327, | |
| "rewards/cosine_scaled_reward": 0.9532012045383453, | |
| "rewards/format_reward": 0.0, | |
| "step": 38 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 857.9230270385742, | |
| "epoch": 0.39543726235741444, | |
| "grad_norm": 0.13035330176353455, | |
| "learning_rate": 9.71519026398956e-07, | |
| "loss": 0.0268, | |
| "num_tokens": 28067040.0, | |
| "reward": 0.9902723953127861, | |
| "reward_std": 0.5246665701270103, | |
| "rewards/cosine_scaled_reward": 0.9902723580598831, | |
| "rewards/format_reward": 0.0, | |
| "step": 39 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 793.2377471923828, | |
| "epoch": 0.4055766793409379, | |
| "grad_norm": 0.13015364110469818, | |
| "learning_rate": 9.68474862499881e-07, | |
| "loss": 0.0268, | |
| "num_tokens": 28897045.0, | |
| "reward": 1.063771203160286, | |
| "reward_std": 0.49807993322610855, | |
| "rewards/cosine_scaled_reward": 1.0637711435556412, | |
| "rewards/format_reward": 0.0, | |
| "step": 40 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 776.3783798217773, | |
| "epoch": 0.4157160963244613, | |
| "grad_norm": 0.12312573939561844, | |
| "learning_rate": 9.652814366302568e-07, | |
| "loss": 0.0136, | |
| "num_tokens": 29718656.0, | |
| "reward": 1.1480179950594902, | |
| "reward_std": 0.5026725828647614, | |
| "rewards/cosine_scaled_reward": 1.1480179578065872, | |
| "rewards/format_reward": 0.0, | |
| "step": 41 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 782.9832916259766, | |
| "epoch": 0.42585551330798477, | |
| "grad_norm": 0.12757915258407593, | |
| "learning_rate": 9.619397662556433e-07, | |
| "loss": 0.0089, | |
| "num_tokens": 30549177.0, | |
| "reward": 1.153884269297123, | |
| "reward_std": 0.43890631198883057, | |
| "rewards/cosine_scaled_reward": 1.1538842469453812, | |
| "rewards/format_reward": 0.0, | |
| "step": 42 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 855.8906631469727, | |
| "epoch": 0.43599493029150826, | |
| "grad_norm": 0.11601021140813828, | |
| "learning_rate": 9.5845091607416e-07, | |
| "loss": -0.0027, | |
| "num_tokens": 31434759.0, | |
| "reward": 1.1496833115816116, | |
| "reward_std": 0.44448646157979965, | |
| "rewards/cosine_scaled_reward": 1.1496832445263863, | |
| "rewards/format_reward": 0.0, | |
| "step": 43 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 862.2332916259766, | |
| "epoch": 0.4461343472750317, | |
| "grad_norm": 0.11746807396411896, | |
| "learning_rate": 9.548159976772592e-07, | |
| "loss": 0.0197, | |
| "num_tokens": 32337368.0, | |
| "reward": 1.1307310312986374, | |
| "reward_std": 0.4776446260511875, | |
| "rewards/cosine_scaled_reward": 1.130730964243412, | |
| "rewards/format_reward": 0.0, | |
| "step": 44 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 900.1975860595703, | |
| "epoch": 0.45627376425855515, | |
| "grad_norm": 0.1248885914683342, | |
| "learning_rate": 9.510361691955606e-07, | |
| "loss": 0.0165, | |
| "num_tokens": 33266393.0, | |
| "reward": 1.060933604836464, | |
| "reward_std": 0.46989936381578445, | |
| "rewards/cosine_scaled_reward": 1.060933567583561, | |
| "rewards/format_reward": 0.0, | |
| "step": 45 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 815.2812881469727, | |
| "epoch": 0.4664131812420786, | |
| "grad_norm": 0.9569998979568481, | |
| "learning_rate": 9.471126349298556e-07, | |
| "loss": 0.0354, | |
| "num_tokens": 34127101.0, | |
| "reward": 1.151917465031147, | |
| "reward_std": 0.5036191828548908, | |
| "rewards/cosine_scaled_reward": 1.1519174352288246, | |
| "rewards/format_reward": 0.0, | |
| "step": 46 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 833.6239242553711, | |
| "epoch": 0.47655259822560203, | |
| "grad_norm": 0.15996558964252472, | |
| "learning_rate": 9.430466449674013e-07, | |
| "loss": -0.0196, | |
| "num_tokens": 35022380.0, | |
| "reward": 1.0830270573496819, | |
| "reward_std": 0.4072240814566612, | |
| "rewards/cosine_scaled_reward": 1.0830269828438759, | |
| "rewards/format_reward": 0.0, | |
| "step": 47 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 892.449821472168, | |
| "epoch": 0.4866920152091255, | |
| "grad_norm": 0.11413749307394028, | |
| "learning_rate": 9.388394947836278e-07, | |
| "loss": -0.0125, | |
| "num_tokens": 35948751.0, | |
| "reward": 1.0179572254419327, | |
| "reward_std": 0.45433366298675537, | |
| "rewards/cosine_scaled_reward": 1.0179571583867073, | |
| "rewards/format_reward": 0.0, | |
| "step": 48 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 912.2645492553711, | |
| "epoch": 0.4968314321926489, | |
| "grad_norm": 0.11429373919963837, | |
| "learning_rate": 9.344925248293835e-07, | |
| "loss": -0.001, | |
| "num_tokens": 36887148.0, | |
| "reward": 1.1400989294052124, | |
| "reward_std": 0.4401072785258293, | |
| "rewards/cosine_scaled_reward": 1.1400988548994064, | |
| "rewards/format_reward": 0.0, | |
| "step": 49 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 866.2511596679688, | |
| "epoch": 0.5069708491761724, | |
| "grad_norm": 0.11980035156011581, | |
| "learning_rate": 9.300071201038501e-07, | |
| "loss": 0.0215, | |
| "num_tokens": 37789485.0, | |
| "reward": 1.13294617831707, | |
| "reward_std": 0.4442785568535328, | |
| "rewards/cosine_scaled_reward": 1.1329461187124252, | |
| "rewards/format_reward": 0.0, | |
| "step": 50 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 799.482177734375, | |
| "epoch": 0.5171102661596958, | |
| "grad_norm": 0.122144915163517, | |
| "learning_rate": 9.253847097132655e-07, | |
| "loss": 0.0044, | |
| "num_tokens": 38636645.0, | |
| "reward": 1.2101461067795753, | |
| "reward_std": 0.41771573945879936, | |
| "rewards/cosine_scaled_reward": 1.210146002471447, | |
| "rewards/format_reward": 0.0, | |
| "step": 51 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 847.4330749511719, | |
| "epoch": 0.5272496831432193, | |
| "grad_norm": 0.1199701800942421, | |
| "learning_rate": 9.206267664155906e-07, | |
| "loss": 0.0082, | |
| "num_tokens": 39516321.0, | |
| "reward": 1.1085221618413925, | |
| "reward_std": 0.4500999916344881, | |
| "rewards/cosine_scaled_reward": 1.1085221245884895, | |
| "rewards/format_reward": 0.0, | |
| "step": 52 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 873.8102951049805, | |
| "epoch": 0.5373891001267427, | |
| "grad_norm": 0.14303860068321228, | |
| "learning_rate": 9.157348061512726e-07, | |
| "loss": -0.0033, | |
| "num_tokens": 40435335.0, | |
| "reward": 1.1145585402846336, | |
| "reward_std": 0.47080911323428154, | |
| "rewards/cosine_scaled_reward": 1.1145584508776665, | |
| "rewards/format_reward": 0.0, | |
| "step": 53 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 961.9029312133789, | |
| "epoch": 0.5475285171102662, | |
| "grad_norm": 0.11308898031711578, | |
| "learning_rate": 9.107103875602458e-07, | |
| "loss": -0.0123, | |
| "num_tokens": 41415752.0, | |
| "reward": 1.0546733513474464, | |
| "reward_std": 0.41840433329343796, | |
| "rewards/cosine_scaled_reward": 1.054673284292221, | |
| "rewards/format_reward": 0.0, | |
| "step": 54 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 892.6484680175781, | |
| "epoch": 0.5576679340937896, | |
| "grad_norm": 0.12083946913480759, | |
| "learning_rate": 9.055551114853295e-07, | |
| "loss": 0.0043, | |
| "num_tokens": 42345957.0, | |
| "reward": 1.0607689917087555, | |
| "reward_std": 0.4757756106555462, | |
| "rewards/cosine_scaled_reward": 1.0607689544558525, | |
| "rewards/format_reward": 0.0, | |
| "step": 55 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 823.7578506469727, | |
| "epoch": 0.5678073510773131, | |
| "grad_norm": 0.21660108864307404, | |
| "learning_rate": 9.002706204621802e-07, | |
| "loss": 0.0311, | |
| "num_tokens": 43207724.0, | |
| "reward": 1.1668334901332855, | |
| "reward_std": 0.482263408601284, | |
| "rewards/cosine_scaled_reward": 1.1668334528803825, | |
| "rewards/format_reward": 0.0, | |
| "step": 56 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 844.7210159301758, | |
| "epoch": 0.5779467680608364, | |
| "grad_norm": 0.4439857602119446, | |
| "learning_rate": 8.948585981959578e-07, | |
| "loss": 0.0112, | |
| "num_tokens": 44096178.0, | |
| "reward": 1.1872114390134811, | |
| "reward_std": 0.44185368344187737, | |
| "rewards/cosine_scaled_reward": 1.1872113794088364, | |
| "rewards/format_reward": 0.0, | |
| "step": 57 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 832.4553909301758, | |
| "epoch": 0.5880861850443599, | |
| "grad_norm": 0.12814383208751678, | |
| "learning_rate": 8.893207690248775e-07, | |
| "loss": -0.0022, | |
| "num_tokens": 44963122.0, | |
| "reward": 1.1495825573801994, | |
| "reward_std": 0.43896420300006866, | |
| "rewards/cosine_scaled_reward": 1.149582527577877, | |
| "rewards/format_reward": 0.0, | |
| "step": 58 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 819.3661117553711, | |
| "epoch": 0.5982256020278834, | |
| "grad_norm": 0.12473784387111664, | |
| "learning_rate": 8.836588973708128e-07, | |
| "loss": -0.0152, | |
| "num_tokens": 45838866.0, | |
| "reward": 1.101597860455513, | |
| "reward_std": 0.44026929885149, | |
| "rewards/cosine_scaled_reward": 1.1015978381037712, | |
| "rewards/format_reward": 0.0, | |
| "step": 59 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 883.5223617553711, | |
| "epoch": 0.6083650190114068, | |
| "grad_norm": 0.12360174208879471, | |
| "learning_rate": 8.778747871771291e-07, | |
| "loss": 0.0119, | |
| "num_tokens": 46767750.0, | |
| "reward": 1.1120553016662598, | |
| "reward_std": 0.43890899419784546, | |
| "rewards/cosine_scaled_reward": 1.1120552495121956, | |
| "rewards/format_reward": 0.0, | |
| "step": 60 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 830.4174499511719, | |
| "epoch": 0.6185044359949303, | |
| "grad_norm": 0.12516207993030548, | |
| "learning_rate": 8.719702813339247e-07, | |
| "loss": 0.007, | |
| "num_tokens": 47634148.0, | |
| "reward": 1.1350341737270355, | |
| "reward_std": 0.3932863809168339, | |
| "rewards/cosine_scaled_reward": 1.1350341215729713, | |
| "rewards/format_reward": 0.0, | |
| "step": 61 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 860.8895492553711, | |
| "epoch": 0.6286438529784537, | |
| "grad_norm": 0.11765479296445847, | |
| "learning_rate": 8.659472610908627e-07, | |
| "loss": -0.0127, | |
| "num_tokens": 48534897.0, | |
| "reward": 1.0959327295422554, | |
| "reward_std": 0.43786268681287766, | |
| "rewards/cosine_scaled_reward": 1.09593266248703, | |
| "rewards/format_reward": 0.0, | |
| "step": 62 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 789.957633972168, | |
| "epoch": 0.6387832699619772, | |
| "grad_norm": 0.12538108229637146, | |
| "learning_rate": 8.598076454577814e-07, | |
| "loss": 0.0043, | |
| "num_tokens": 49366995.0, | |
| "reward": 1.2128012925386429, | |
| "reward_std": 0.5132176093757153, | |
| "rewards/cosine_scaled_reward": 1.2128012254834175, | |
| "rewards/format_reward": 0.0, | |
| "step": 63 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 814.1540603637695, | |
| "epoch": 0.6489226869455006, | |
| "grad_norm": 0.12521201372146606, | |
| "learning_rate": 8.535533905932737e-07, | |
| "loss": 0.031, | |
| "num_tokens": 50244605.0, | |
| "reward": 1.2290623039007187, | |
| "reward_std": 0.35417212545871735, | |
| "rewards/cosine_scaled_reward": 1.2290622666478157, | |
| "rewards/format_reward": 0.0, | |
| "step": 64 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 826.7444610595703, | |
| "epoch": 0.6590621039290241, | |
| "grad_norm": 0.1192706748843193, | |
| "learning_rate": 8.471864891814304e-07, | |
| "loss": 0.0187, | |
| "num_tokens": 51104472.0, | |
| "reward": 1.2224428355693817, | |
| "reward_std": 0.4238445721566677, | |
| "rewards/cosine_scaled_reward": 1.2224428057670593, | |
| "rewards/format_reward": 0.0, | |
| "step": 65 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 890.8181228637695, | |
| "epoch": 0.6692015209125475, | |
| "grad_norm": 0.11663277447223663, | |
| "learning_rate": 8.407089697969456e-07, | |
| "loss": -0.0075, | |
| "num_tokens": 52027509.0, | |
| "reward": 1.1307330876588821, | |
| "reward_std": 0.43055112659931183, | |
| "rewards/cosine_scaled_reward": 1.130733035504818, | |
| "rewards/format_reward": 0.0, | |
| "step": 66 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 896.5524978637695, | |
| "epoch": 0.679340937896071, | |
| "grad_norm": 0.11125318706035614, | |
| "learning_rate": 8.341228962587881e-07, | |
| "loss": -0.0128, | |
| "num_tokens": 52966948.0, | |
| "reward": 1.1304329186677933, | |
| "reward_std": 0.38643651083111763, | |
| "rewards/cosine_scaled_reward": 1.1304328739643097, | |
| "rewards/format_reward": 0.0, | |
| "step": 67 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 805.1919937133789, | |
| "epoch": 0.6894803548795945, | |
| "grad_norm": 0.1318301111459732, | |
| "learning_rate": 8.274303669726426e-07, | |
| "loss": 0.0267, | |
| "num_tokens": 53809720.0, | |
| "reward": 1.2848119288682938, | |
| "reward_std": 0.4024609997868538, | |
| "rewards/cosine_scaled_reward": 1.2848118543624878, | |
| "rewards/format_reward": 0.0, | |
| "step": 68 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 897.3783950805664, | |
| "epoch": 0.6996197718631179, | |
| "grad_norm": 0.11534889042377472, | |
| "learning_rate": 8.206335142623304e-07, | |
| "loss": -0.0213, | |
| "num_tokens": 54743531.0, | |
| "reward": 1.0870068296790123, | |
| "reward_std": 0.396763913333416, | |
| "rewards/cosine_scaled_reward": 1.0870067551732063, | |
| "rewards/format_reward": 0.0, | |
| "step": 69 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 849.7935562133789, | |
| "epoch": 0.7097591888466414, | |
| "grad_norm": 0.11737702041864395, | |
| "learning_rate": 8.137345036904259e-07, | |
| "loss": 0.0049, | |
| "num_tokens": 55624322.0, | |
| "reward": 1.1217299401760101, | |
| "reward_std": 0.4600865840911865, | |
| "rewards/cosine_scaled_reward": 1.1217298731207848, | |
| "rewards/format_reward": 0.0, | |
| "step": 70 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 843.4699020385742, | |
| "epoch": 0.7198986058301647, | |
| "grad_norm": 0.12105900794267654, | |
| "learning_rate": 8.067355333682797e-07, | |
| "loss": 0.0058, | |
| "num_tokens": 56518839.0, | |
| "reward": 1.128207340836525, | |
| "reward_std": 0.39387310668826103, | |
| "rewards/cosine_scaled_reward": 1.1282072588801384, | |
| "rewards/format_reward": 0.0, | |
| "step": 71 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 854.9007110595703, | |
| "epoch": 0.7300380228136882, | |
| "grad_norm": 0.11754778772592545, | |
| "learning_rate": 7.996388332556734e-07, | |
| "loss": 0.0184, | |
| "num_tokens": 57413550.0, | |
| "reward": 1.1404274106025696, | |
| "reward_std": 0.45089153945446014, | |
| "rewards/cosine_scaled_reward": 1.1404273584485054, | |
| "rewards/format_reward": 0.0, | |
| "step": 72 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 900.8973541259766, | |
| "epoch": 0.7401774397972116, | |
| "grad_norm": 0.11590257287025452, | |
| "learning_rate": 7.924466644503264e-07, | |
| "loss": -0.0059, | |
| "num_tokens": 58353250.0, | |
| "reward": 1.1319697871804237, | |
| "reward_std": 0.4219101257622242, | |
| "rewards/cosine_scaled_reward": 1.131969727575779, | |
| "rewards/format_reward": 0.0, | |
| "step": 73 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 806.4665603637695, | |
| "epoch": 0.7503168567807351, | |
| "grad_norm": 0.12108156830072403, | |
| "learning_rate": 7.85161318467482e-07, | |
| "loss": -0.0001, | |
| "num_tokens": 59211228.0, | |
| "reward": 1.1962338984012604, | |
| "reward_std": 0.45812737941741943, | |
| "rewards/cosine_scaled_reward": 1.196233868598938, | |
| "rewards/format_reward": 0.0, | |
| "step": 74 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 812.6629791259766, | |
| "epoch": 0.7604562737642585, | |
| "grad_norm": 0.12231780588626862, | |
| "learning_rate": 7.777851165098011e-07, | |
| "loss": 0.0223, | |
| "num_tokens": 60070054.0, | |
| "reward": 1.2705483883619308, | |
| "reward_std": 0.4315441697835922, | |
| "rewards/cosine_scaled_reward": 1.2705483138561249, | |
| "rewards/format_reward": 0.0, | |
| "step": 75 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 792.5323944091797, | |
| "epoch": 0.770595690747782, | |
| "grad_norm": 0.13156548142433167, | |
| "learning_rate": 7.703204087277988e-07, | |
| "loss": 0.0054, | |
| "num_tokens": 60907403.0, | |
| "reward": 1.0341035649180412, | |
| "reward_std": 0.4903283417224884, | |
| "rewards/cosine_scaled_reward": 1.0341035276651382, | |
| "rewards/format_reward": 0.0, | |
| "step": 76 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 843.8315200805664, | |
| "epoch": 0.7807351077313055, | |
| "grad_norm": 0.12101699411869049, | |
| "learning_rate": 7.627695734710564e-07, | |
| "loss": 0.0109, | |
| "num_tokens": 61785700.0, | |
| "reward": 1.226922646164894, | |
| "reward_std": 0.38846197351813316, | |
| "rewards/cosine_scaled_reward": 1.2269226014614105, | |
| "rewards/format_reward": 0.0, | |
| "step": 77 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 859.2433395385742, | |
| "epoch": 0.7908745247148289, | |
| "grad_norm": 0.1146969199180603, | |
| "learning_rate": 7.551350165304499e-07, | |
| "loss": 0.0219, | |
| "num_tokens": 62673358.0, | |
| "reward": 1.2758738100528717, | |
| "reward_std": 0.4126880206167698, | |
| "rewards/cosine_scaled_reward": 1.275873750448227, | |
| "rewards/format_reward": 0.0, | |
| "step": 78 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 847.4732513427734, | |
| "epoch": 0.8010139416983524, | |
| "grad_norm": 0.11725428700447083, | |
| "learning_rate": 7.474191703716338e-07, | |
| "loss": -0.0134, | |
| "num_tokens": 63563254.0, | |
| "reward": 1.1727145612239838, | |
| "reward_std": 0.3684752322733402, | |
| "rewards/cosine_scaled_reward": 1.1727144792675972, | |
| "rewards/format_reward": 0.0, | |
| "step": 79 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 831.0089721679688, | |
| "epoch": 0.8111533586818758, | |
| "grad_norm": 0.1215280145406723, | |
| "learning_rate": 7.396244933600284e-07, | |
| "loss": 0.0269, | |
| "num_tokens": 64432126.0, | |
| "reward": 1.1777353882789612, | |
| "reward_std": 0.420895554125309, | |
| "rewards/cosine_scaled_reward": 1.1777353659272194, | |
| "rewards/format_reward": 0.0, | |
| "step": 80 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 848.4598617553711, | |
| "epoch": 0.8212927756653993, | |
| "grad_norm": 0.13223014771938324, | |
| "learning_rate": 7.317534689775527e-07, | |
| "loss": 0.0072, | |
| "num_tokens": 65312050.0, | |
| "reward": 1.1698129922151566, | |
| "reward_std": 0.4710509404540062, | |
| "rewards/cosine_scaled_reward": 1.1698129400610924, | |
| "rewards/format_reward": 0.0, | |
| "step": 81 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 811.4230346679688, | |
| "epoch": 0.8314321926489227, | |
| "grad_norm": 0.12015990167856216, | |
| "learning_rate": 7.238086050313562e-07, | |
| "loss": -0.0264, | |
| "num_tokens": 66174341.0, | |
| "reward": 1.124290645122528, | |
| "reward_std": 0.40254683420062065, | |
| "rewards/cosine_scaled_reward": 1.1242906004190445, | |
| "rewards/format_reward": 0.0, | |
| "step": 82 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 819.8382110595703, | |
| "epoch": 0.8415716096324461, | |
| "grad_norm": 0.14578643441200256, | |
| "learning_rate": 7.157924328548002e-07, | |
| "loss": 0.026, | |
| "num_tokens": 67042508.0, | |
| "reward": 1.2035124003887177, | |
| "reward_std": 0.4501505568623543, | |
| "rewards/cosine_scaled_reward": 1.2035123333334923, | |
| "rewards/format_reward": 0.0, | |
| "step": 83 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 832.864990234375, | |
| "epoch": 0.8517110266159695, | |
| "grad_norm": 0.11915823817253113, | |
| "learning_rate": 7.077075065009433e-07, | |
| "loss": 0.0419, | |
| "num_tokens": 67922067.0, | |
| "reward": 1.2239351123571396, | |
| "reward_std": 0.4415438733994961, | |
| "rewards/cosine_scaled_reward": 1.2239350602030754, | |
| "rewards/format_reward": 0.0, | |
| "step": 84 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 794.1897659301758, | |
| "epoch": 0.861850443599493, | |
| "grad_norm": 0.12470311671495438, | |
| "learning_rate": 6.995564019287869e-07, | |
| "loss": 0.0085, | |
| "num_tokens": 68763477.0, | |
| "reward": 1.1386212185025215, | |
| "reward_std": 0.4416712336242199, | |
| "rewards/cosine_scaled_reward": 1.1386211588978767, | |
| "rewards/format_reward": 0.0, | |
| "step": 85 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 865.2344207763672, | |
| "epoch": 0.8719898605830165, | |
| "grad_norm": 0.1167558878660202, | |
| "learning_rate": 6.913417161825449e-07, | |
| "loss": -0.0029, | |
| "num_tokens": 69673639.0, | |
| "reward": 1.1339842602610588, | |
| "reward_std": 0.44360488280653954, | |
| "rewards/cosine_scaled_reward": 1.1339841783046722, | |
| "rewards/format_reward": 0.0, | |
| "step": 86 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 807.192008972168, | |
| "epoch": 0.8821292775665399, | |
| "grad_norm": 0.22383332252502441, | |
| "learning_rate": 6.830660665641897e-07, | |
| "loss": 0.0167, | |
| "num_tokens": 70526603.0, | |
| "reward": 1.1783367395401, | |
| "reward_std": 0.44941645860671997, | |
| "rewards/cosine_scaled_reward": 1.1783367022871971, | |
| "rewards/format_reward": 0.0, | |
| "step": 87 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 842.9832916259766, | |
| "epoch": 0.8922686945500634, | |
| "grad_norm": 0.12214665859937668, | |
| "learning_rate": 6.747320897995492e-07, | |
| "loss": 0.0028, | |
| "num_tokens": 71410908.0, | |
| "reward": 1.1979937851428986, | |
| "reward_std": 0.41686391085386276, | |
| "rewards/cosine_scaled_reward": 1.1979937255382538, | |
| "rewards/format_reward": 0.0, | |
| "step": 88 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 752.7991409301758, | |
| "epoch": 0.9024081115335868, | |
| "grad_norm": 0.12918852269649506, | |
| "learning_rate": 6.66342441198212e-07, | |
| "loss": -0.0199, | |
| "num_tokens": 72209320.0, | |
| "reward": 1.1556047648191452, | |
| "reward_std": 0.39559851959347725, | |
| "rewards/cosine_scaled_reward": 1.1556047424674034, | |
| "rewards/format_reward": 0.0, | |
| "step": 89 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 788.5815048217773, | |
| "epoch": 0.9125475285171103, | |
| "grad_norm": 0.2962648272514343, | |
| "learning_rate": 6.578997938075125e-07, | |
| "loss": 0.0258, | |
| "num_tokens": 73040361.0, | |
| "reward": 1.1919841021299362, | |
| "reward_std": 0.40032988227903843, | |
| "rewards/cosine_scaled_reward": 1.1919840648770332, | |
| "rewards/format_reward": 0.0, | |
| "step": 90 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 787.3214645385742, | |
| "epoch": 0.9226869455006337, | |
| "grad_norm": 0.15694542229175568, | |
| "learning_rate": 6.494068375608646e-07, | |
| "loss": -0.0147, | |
| "num_tokens": 73880425.0, | |
| "reward": 1.1829868927598, | |
| "reward_std": 0.3718248065561056, | |
| "rewards/cosine_scaled_reward": 1.1829868257045746, | |
| "rewards/format_reward": 0.0, | |
| "step": 91 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 821.3850708007812, | |
| "epoch": 0.9328263624841572, | |
| "grad_norm": 0.11674854159355164, | |
| "learning_rate": 6.408662784207149e-07, | |
| "loss": 0.0095, | |
| "num_tokens": 74755762.0, | |
| "reward": 1.2243514209985733, | |
| "reward_std": 0.4170701913535595, | |
| "rewards/cosine_scaled_reward": 1.2243513762950897, | |
| "rewards/format_reward": 0.0, | |
| "step": 92 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 761.3248138427734, | |
| "epoch": 0.9429657794676806, | |
| "grad_norm": 0.12954537570476532, | |
| "learning_rate": 6.322808375163895e-07, | |
| "loss": 0.0386, | |
| "num_tokens": 75564789.0, | |
| "reward": 1.1891243010759354, | |
| "reward_std": 0.45389125496149063, | |
| "rewards/cosine_scaled_reward": 1.1891242563724518, | |
| "rewards/format_reward": 0.0, | |
| "step": 93 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 793.9643249511719, | |
| "epoch": 0.9531051964512041, | |
| "grad_norm": 0.12744048237800598, | |
| "learning_rate": 6.236532502771077e-07, | |
| "loss": 0.0271, | |
| "num_tokens": 76409781.0, | |
| "reward": 1.1992642730474472, | |
| "reward_std": 0.4242793843150139, | |
| "rewards/cosine_scaled_reward": 1.1992641612887383, | |
| "rewards/format_reward": 0.0, | |
| "step": 94 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 789.4219207763672, | |
| "epoch": 0.9632446134347274, | |
| "grad_norm": 0.12654094398021698, | |
| "learning_rate": 6.149862655604403e-07, | |
| "loss": 0.0339, | |
| "num_tokens": 77242791.0, | |
| "reward": 1.2284981906414032, | |
| "reward_std": 0.42066042125225067, | |
| "rewards/cosine_scaled_reward": 1.2284981459379196, | |
| "rewards/format_reward": 0.0, | |
| "step": 95 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 795.716552734375, | |
| "epoch": 0.973384030418251, | |
| "grad_norm": 0.12548381090164185, | |
| "learning_rate": 6.062826447764883e-07, | |
| "loss": 0.008, | |
| "num_tokens": 78078497.0, | |
| "reward": 1.2442717999219894, | |
| "reward_std": 0.35396804101765156, | |
| "rewards/cosine_scaled_reward": 1.2442717254161835, | |
| "rewards/format_reward": 0.0, | |
| "step": 96 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 793.0692291259766, | |
| "epoch": 0.9835234474017744, | |
| "grad_norm": 0.12744414806365967, | |
| "learning_rate": 5.975451610080642e-07, | |
| "loss": -0.0194, | |
| "num_tokens": 78916895.0, | |
| "reward": 1.1445267572999, | |
| "reward_std": 0.37857919186353683, | |
| "rewards/cosine_scaled_reward": 1.1445267051458359, | |
| "rewards/format_reward": 0.0, | |
| "step": 97 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 782.1171875, | |
| "epoch": 0.9936628643852978, | |
| "grad_norm": 0.12226420640945435, | |
| "learning_rate": 5.887765981271517e-07, | |
| "loss": 0.0066, | |
| "num_tokens": 79784779.0, | |
| "reward": 1.168272152543068, | |
| "reward_std": 0.39856788516044617, | |
| "rewards/cosine_scaled_reward": 1.1682721078395844, | |
| "rewards/format_reward": 0.0, | |
| "step": 98 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 783.4386367797852, | |
| "epoch": 1.0101394169835234, | |
| "grad_norm": 0.13228166103363037, | |
| "learning_rate": 5.7997974990793e-07, | |
| "loss": 0.0145, | |
| "num_tokens": 80615732.0, | |
| "reward": 1.2336678951978683, | |
| "reward_std": 0.38141428492963314, | |
| "rewards/cosine_scaled_reward": 1.233667865395546, | |
| "rewards/format_reward": 0.0, | |
| "step": 99 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 806.2355346679688, | |
| "epoch": 1.020278833967047, | |
| "grad_norm": 0.12670038640499115, | |
| "learning_rate": 5.711574191366427e-07, | |
| "loss": 0.0028, | |
| "num_tokens": 81465495.0, | |
| "reward": 1.1623064577579498, | |
| "reward_std": 0.3932801876217127, | |
| "rewards/cosine_scaled_reward": 1.1623064130544662, | |
| "rewards/format_reward": 0.0, | |
| "step": 100 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 790.1205749511719, | |
| "epoch": 1.0304182509505704, | |
| "grad_norm": 0.12086564302444458, | |
| "learning_rate": 5.623124167185929e-07, | |
| "loss": 0.0063, | |
| "num_tokens": 82295971.0, | |
| "reward": 1.2574269622564316, | |
| "reward_std": 0.3898525554686785, | |
| "rewards/cosine_scaled_reward": 1.257426917552948, | |
| "rewards/format_reward": 0.0, | |
| "step": 101 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 758.897346496582, | |
| "epoch": 1.0405576679340938, | |
| "grad_norm": 0.14097033441066742, | |
| "learning_rate": 5.534475607825565e-07, | |
| "loss": -0.0126, | |
| "num_tokens": 83103231.0, | |
| "reward": 1.2127035036683083, | |
| "reward_std": 0.3951477538794279, | |
| "rewards/cosine_scaled_reward": 1.2127034589648247, | |
| "rewards/format_reward": 0.0, | |
| "step": 102 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 823.2377624511719, | |
| "epoch": 1.0506970849176172, | |
| "grad_norm": 0.11900278925895691, | |
| "learning_rate": 5.445656757828879e-07, | |
| "loss": -0.0341, | |
| "num_tokens": 83972156.0, | |
| "reward": 1.0786512047052383, | |
| "reward_std": 0.386748855933547, | |
| "rewards/cosine_scaled_reward": 1.078651174902916, | |
| "rewards/format_reward": 0.0, | |
| "step": 103 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 778.1082992553711, | |
| "epoch": 1.0608365019011408, | |
| "grad_norm": 0.12485373765230179, | |
| "learning_rate": 5.356695915996161e-07, | |
| "loss": -0.0152, | |
| "num_tokens": 84803701.0, | |
| "reward": 1.1336441561579704, | |
| "reward_std": 0.43268290534615517, | |
| "rewards/cosine_scaled_reward": 1.1336440965533257, | |
| "rewards/format_reward": 0.0, | |
| "step": 104 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 747.7790451049805, | |
| "epoch": 1.0709759188846641, | |
| "grad_norm": 0.14275644719600677, | |
| "learning_rate": 5.267621426368075e-07, | |
| "loss": 0.0402, | |
| "num_tokens": 85592927.0, | |
| "reward": 1.2611572295427322, | |
| "reward_std": 0.4418743886053562, | |
| "rewards/cosine_scaled_reward": 1.2611571997404099, | |
| "rewards/format_reward": 0.0, | |
| "step": 105 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 686.8861770629883, | |
| "epoch": 1.0811153358681875, | |
| "grad_norm": 0.2392619252204895, | |
| "learning_rate": 5.178461669194903e-07, | |
| "loss": 0.0233, | |
| "num_tokens": 86335585.0, | |
| "reward": 1.2902618646621704, | |
| "reward_std": 0.4318929500877857, | |
| "rewards/cosine_scaled_reward": 1.2902617901563644, | |
| "rewards/format_reward": 0.0, | |
| "step": 106 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 765.6752548217773, | |
| "epoch": 1.091254752851711, | |
| "grad_norm": 0.12888778746128082, | |
| "learning_rate": 5.08924505189423e-07, | |
| "loss": -0.0053, | |
| "num_tokens": 87147590.0, | |
| "reward": 1.2406404912471771, | |
| "reward_std": 0.3950204625725746, | |
| "rewards/cosine_scaled_reward": 1.2406404465436935, | |
| "rewards/format_reward": 0.0, | |
| "step": 107 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 774.6596298217773, | |
| "epoch": 1.1013941698352345, | |
| "grad_norm": 0.16063496470451355, | |
| "learning_rate": 5e-07, | |
| "loss": 0.0407, | |
| "num_tokens": 87965805.0, | |
| "reward": 1.2785077691078186, | |
| "reward_std": 0.39564304798841476, | |
| "rewards/cosine_scaled_reward": 1.2785076946020126, | |
| "rewards/format_reward": 0.0, | |
| "step": 108 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 800.3393249511719, | |
| "epoch": 1.111533586818758, | |
| "grad_norm": 0.23322370648384094, | |
| "learning_rate": 4.91075494810577e-07, | |
| "loss": 0.0181, | |
| "num_tokens": 88800901.0, | |
| "reward": 1.2374378964304924, | |
| "reward_std": 0.39184754714369774, | |
| "rewards/cosine_scaled_reward": 1.2374378442764282, | |
| "rewards/format_reward": 0.0, | |
| "step": 109 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 802.3839721679688, | |
| "epoch": 1.1216730038022813, | |
| "grad_norm": 0.12637439370155334, | |
| "learning_rate": 4.821538330805098e-07, | |
| "loss": -0.0142, | |
| "num_tokens": 89646357.0, | |
| "reward": 1.2594422698020935, | |
| "reward_std": 0.3456545788794756, | |
| "rewards/cosine_scaled_reward": 1.2594422399997711, | |
| "rewards/format_reward": 0.0, | |
| "step": 110 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 804.2109680175781, | |
| "epoch": 1.131812420785805, | |
| "grad_norm": 0.4338585138320923, | |
| "learning_rate": 4.732378573631924e-07, | |
| "loss": -0.0124, | |
| "num_tokens": 90489658.0, | |
| "reward": 1.204567864537239, | |
| "reward_std": 0.44952966272830963, | |
| "rewards/cosine_scaled_reward": 1.2045678049325943, | |
| "rewards/format_reward": 0.0, | |
| "step": 111 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 814.7243728637695, | |
| "epoch": 1.1419518377693283, | |
| "grad_norm": 0.12191635370254517, | |
| "learning_rate": 4.643304084003838e-07, | |
| "loss": 0.0138, | |
| "num_tokens": 91347099.0, | |
| "reward": 1.1750361323356628, | |
| "reward_std": 0.4308737814426422, | |
| "rewards/cosine_scaled_reward": 1.1750360652804375, | |
| "rewards/format_reward": 0.0, | |
| "step": 112 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 767.3794937133789, | |
| "epoch": 1.1520912547528517, | |
| "grad_norm": 0.13367751240730286, | |
| "learning_rate": 4.55434324217112e-07, | |
| "loss": -0.026, | |
| "num_tokens": 92163911.0, | |
| "reward": 1.0875676944851875, | |
| "reward_std": 0.4285357743501663, | |
| "rewards/cosine_scaled_reward": 1.0875676572322845, | |
| "rewards/format_reward": 0.0, | |
| "step": 113 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 759.1138687133789, | |
| "epoch": 1.162230671736375, | |
| "grad_norm": 0.6044357419013977, | |
| "learning_rate": 4.4655243921744367e-07, | |
| "loss": -0.0034, | |
| "num_tokens": 92969037.0, | |
| "reward": 1.1716800779104233, | |
| "reward_std": 0.40124501287937164, | |
| "rewards/cosine_scaled_reward": 1.1716800406575203, | |
| "rewards/format_reward": 0.0, | |
| "step": 114 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 742.3248138427734, | |
| "epoch": 1.1723700887198987, | |
| "grad_norm": 0.12603311240673065, | |
| "learning_rate": 4.37687583281407e-07, | |
| "loss": -0.0106, | |
| "num_tokens": 93761376.0, | |
| "reward": 1.2006227523088455, | |
| "reward_std": 0.3855649037286639, | |
| "rewards/cosine_scaled_reward": 1.2006226852536201, | |
| "rewards/format_reward": 0.0, | |
| "step": 115 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 750.4765930175781, | |
| "epoch": 1.182509505703422, | |
| "grad_norm": 0.13189645111560822, | |
| "learning_rate": 4.2884258086335745e-07, | |
| "loss": 0.0114, | |
| "num_tokens": 94573043.0, | |
| "reward": 1.2511205673217773, | |
| "reward_std": 0.408273559063673, | |
| "rewards/cosine_scaled_reward": 1.2511205524206161, | |
| "rewards/format_reward": 0.0, | |
| "step": 116 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 748.3750305175781, | |
| "epoch": 1.1926489226869454, | |
| "grad_norm": 0.13386943936347961, | |
| "learning_rate": 4.2002025009206987e-07, | |
| "loss": 0.0081, | |
| "num_tokens": 95375539.0, | |
| "reward": 1.1935091465711594, | |
| "reward_std": 0.39292260631918907, | |
| "rewards/cosine_scaled_reward": 1.1935090869665146, | |
| "rewards/format_reward": 0.0, | |
| "step": 117 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 752.021240234375, | |
| "epoch": 1.202788339670469, | |
| "grad_norm": 0.13410834968090057, | |
| "learning_rate": 4.1122340187284845e-07, | |
| "loss": -0.0055, | |
| "num_tokens": 96171638.0, | |
| "reward": 1.215317651629448, | |
| "reward_std": 0.4204988572746515, | |
| "rewards/cosine_scaled_reward": 1.2153176292777061, | |
| "rewards/format_reward": 0.0, | |
| "step": 118 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 741.1294937133789, | |
| "epoch": 1.2129277566539924, | |
| "grad_norm": 0.14352215826511383, | |
| "learning_rate": 4.0245483899193586e-07, | |
| "loss": -0.0021, | |
| "num_tokens": 96963066.0, | |
| "reward": 1.171108528971672, | |
| "reward_std": 0.390108622610569, | |
| "rewards/cosine_scaled_reward": 1.171108491718769, | |
| "rewards/format_reward": 0.0, | |
| "step": 119 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 731.9676666259766, | |
| "epoch": 1.2230671736375158, | |
| "grad_norm": 0.3331538140773773, | |
| "learning_rate": 3.937173552235116e-07, | |
| "loss": -0.0142, | |
| "num_tokens": 97751717.0, | |
| "reward": 1.1743304282426834, | |
| "reward_std": 0.3502536006271839, | |
| "rewards/cosine_scaled_reward": 1.174330398440361, | |
| "rewards/format_reward": 0.0, | |
| "step": 120 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 722.037971496582, | |
| "epoch": 1.2332065906210392, | |
| "grad_norm": 0.5637516975402832, | |
| "learning_rate": 3.850137344395598e-07, | |
| "loss": 0.0232, | |
| "num_tokens": 98521879.0, | |
| "reward": 1.261700451374054, | |
| "reward_std": 0.34792254120111465, | |
| "rewards/cosine_scaled_reward": 1.2617004364728928, | |
| "rewards/format_reward": 0.0, | |
| "step": 121 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 810.3984832763672, | |
| "epoch": 1.2433460076045628, | |
| "grad_norm": 0.12412155419588089, | |
| "learning_rate": 3.763467497228922e-07, | |
| "loss": -0.0175, | |
| "num_tokens": 99386028.0, | |
| "reward": 1.1121523007750511, | |
| "reward_std": 0.4185631051659584, | |
| "rewards/cosine_scaled_reward": 1.11215228587389, | |
| "rewards/format_reward": 0.0, | |
| "step": 122 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 762.1272659301758, | |
| "epoch": 1.2534854245880862, | |
| "grad_norm": 0.5518940091133118, | |
| "learning_rate": 3.677191624836106e-07, | |
| "loss": -0.0156, | |
| "num_tokens": 100190606.0, | |
| "reward": 1.2098966389894485, | |
| "reward_std": 0.42287516221404076, | |
| "rewards/cosine_scaled_reward": 1.2098965868353844, | |
| "rewards/format_reward": 0.0, | |
| "step": 123 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 797.1741409301758, | |
| "epoch": 1.2636248415716096, | |
| "grad_norm": 0.13090519607067108, | |
| "learning_rate": 3.591337215792851e-07, | |
| "loss": -0.0049, | |
| "num_tokens": 101033042.0, | |
| "reward": 1.2026162892580032, | |
| "reward_std": 0.38727976381778717, | |
| "rewards/cosine_scaled_reward": 1.2026162147521973, | |
| "rewards/format_reward": 0.0, | |
| "step": 124 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 755.0368728637695, | |
| "epoch": 1.2737642585551332, | |
| "grad_norm": 0.13797762989997864, | |
| "learning_rate": 3.505931624391355e-07, | |
| "loss": 0.0129, | |
| "num_tokens": 101827819.0, | |
| "reward": 1.1926257461309433, | |
| "reward_std": 0.44687049090862274, | |
| "rewards/cosine_scaled_reward": 1.1926256865262985, | |
| "rewards/format_reward": 0.0, | |
| "step": 125 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 741.3024978637695, | |
| "epoch": 1.2839036755386566, | |
| "grad_norm": 0.18938252329826355, | |
| "learning_rate": 3.421002061924876e-07, | |
| "loss": 0.0201, | |
| "num_tokens": 102618890.0, | |
| "reward": 1.263178527355194, | |
| "reward_std": 0.3987303748726845, | |
| "rewards/cosine_scaled_reward": 1.2631784602999687, | |
| "rewards/format_reward": 0.0, | |
| "step": 126 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 747.1696701049805, | |
| "epoch": 1.29404309252218, | |
| "grad_norm": 0.13063876330852509, | |
| "learning_rate": 3.3365755880178807e-07, | |
| "loss": 0.0312, | |
| "num_tokens": 103410058.0, | |
| "reward": 1.2827118784189224, | |
| "reward_std": 0.3673780020326376, | |
| "rewards/cosine_scaled_reward": 1.2827118039131165, | |
| "rewards/format_reward": 0.0, | |
| "step": 127 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 796.0893173217773, | |
| "epoch": 1.3041825095057034, | |
| "grad_norm": 0.12410367280244827, | |
| "learning_rate": 3.2526791020045087e-07, | |
| "loss": 0.0153, | |
| "num_tokens": 104255954.0, | |
| "reward": 1.13737141340971, | |
| "reward_std": 0.43270402774214745, | |
| "rewards/cosine_scaled_reward": 1.1373713836073875, | |
| "rewards/format_reward": 0.0, | |
| "step": 128 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 709.1953353881836, | |
| "epoch": 1.3143219264892267, | |
| "grad_norm": 0.13791009783744812, | |
| "learning_rate": 3.169339334358104e-07, | |
| "loss": 0.0012, | |
| "num_tokens": 105024737.0, | |
| "reward": 1.2293454632163048, | |
| "reward_std": 0.36772353760898113, | |
| "rewards/cosine_scaled_reward": 1.2293454110622406, | |
| "rewards/format_reward": 0.0, | |
| "step": 129 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 754.825927734375, | |
| "epoch": 1.3244613434727504, | |
| "grad_norm": 0.13509666919708252, | |
| "learning_rate": 3.086582838174551e-07, | |
| "loss": -0.0033, | |
| "num_tokens": 105833549.0, | |
| "reward": 1.1948751211166382, | |
| "reward_std": 0.37703990191221237, | |
| "rewards/cosine_scaled_reward": 1.1948750466108322, | |
| "rewards/format_reward": 0.0, | |
| "step": 130 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 752.9855194091797, | |
| "epoch": 1.3346007604562737, | |
| "grad_norm": 0.13409507274627686, | |
| "learning_rate": 3.004435980712129e-07, | |
| "loss": -0.0299, | |
| "num_tokens": 106631688.0, | |
| "reward": 1.2489657402038574, | |
| "reward_std": 0.3884260356426239, | |
| "rewards/cosine_scaled_reward": 1.248965710401535, | |
| "rewards/format_reward": 0.0, | |
| "step": 131 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 728.0178909301758, | |
| "epoch": 1.3447401774397973, | |
| "grad_norm": 0.14096473157405853, | |
| "learning_rate": 2.922924934990568e-07, | |
| "loss": 0.0421, | |
| "num_tokens": 107409144.0, | |
| "reward": 1.2679341062903404, | |
| "reward_std": 0.4113157168030739, | |
| "rewards/cosine_scaled_reward": 1.2679340541362762, | |
| "rewards/format_reward": 0.0, | |
| "step": 132 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 708.2924423217773, | |
| "epoch": 1.3548795944233207, | |
| "grad_norm": 0.13527466356754303, | |
| "learning_rate": 2.8420756714519954e-07, | |
| "loss": 0.0099, | |
| "num_tokens": 108188078.0, | |
| "reward": 1.2067532986402512, | |
| "reward_std": 0.4394964314997196, | |
| "rewards/cosine_scaled_reward": 1.2067532688379288, | |
| "rewards/format_reward": 0.0, | |
| "step": 133 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 745.264533996582, | |
| "epoch": 1.3650190114068441, | |
| "grad_norm": 0.12941937148571014, | |
| "learning_rate": 2.7619139496864376e-07, | |
| "loss": -0.0117, | |
| "num_tokens": 108991819.0, | |
| "reward": 1.1342971697449684, | |
| "reward_std": 0.35589409433305264, | |
| "rewards/cosine_scaled_reward": 1.1342971250414848, | |
| "rewards/format_reward": 0.0, | |
| "step": 134 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 690.6328506469727, | |
| "epoch": 1.3751584283903675, | |
| "grad_norm": 0.1342969536781311, | |
| "learning_rate": 2.6824653102244727e-07, | |
| "loss": 0.0057, | |
| "num_tokens": 109744698.0, | |
| "reward": 1.20839624106884, | |
| "reward_std": 0.44357743114233017, | |
| "rewards/cosine_scaled_reward": 1.208396166563034, | |
| "rewards/format_reward": 0.0, | |
| "step": 135 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 739.3605270385742, | |
| "epoch": 1.385297845373891, | |
| "grad_norm": 0.22325177490711212, | |
| "learning_rate": 2.603755066399718e-07, | |
| "loss": 0.0066, | |
| "num_tokens": 110537245.0, | |
| "reward": 1.1991283893585205, | |
| "reward_std": 0.4422716274857521, | |
| "rewards/cosine_scaled_reward": 1.1991283744573593, | |
| "rewards/format_reward": 0.0, | |
| "step": 136 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 771.5045013427734, | |
| "epoch": 1.3954372623574145, | |
| "grad_norm": 0.131967693567276, | |
| "learning_rate": 2.5258082962836614e-07, | |
| "loss": -0.0609, | |
| "num_tokens": 111363073.0, | |
| "reward": 1.0822726637125015, | |
| "reward_std": 0.31478939671069384, | |
| "rewards/cosine_scaled_reward": 1.082272619009018, | |
| "rewards/format_reward": 0.0, | |
| "step": 137 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 745.6908798217773, | |
| "epoch": 1.4055766793409379, | |
| "grad_norm": 0.13431350886821747, | |
| "learning_rate": 2.4486498346955023e-07, | |
| "loss": 0.0393, | |
| "num_tokens": 112154044.0, | |
| "reward": 1.2083490192890167, | |
| "reward_std": 0.4234139323234558, | |
| "rewards/cosine_scaled_reward": 1.2083489745855331, | |
| "rewards/format_reward": 0.0, | |
| "step": 138 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 730.8303909301758, | |
| "epoch": 1.4157160963244613, | |
| "grad_norm": 0.37255144119262695, | |
| "learning_rate": 2.372304265289436e-07, | |
| "loss": 0.0129, | |
| "num_tokens": 112933164.0, | |
| "reward": 1.14224873483181, | |
| "reward_std": 0.47245585173368454, | |
| "rewards/cosine_scaled_reward": 1.142248660326004, | |
| "rewards/format_reward": 0.0, | |
| "step": 139 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 723.6975708007812, | |
| "epoch": 1.4258555133079849, | |
| "grad_norm": 0.13437899947166443, | |
| "learning_rate": 2.2967959127220137e-07, | |
| "loss": 0.0149, | |
| "num_tokens": 113703557.0, | |
| "reward": 1.2409879192709923, | |
| "reward_std": 0.39572571590542793, | |
| "rewards/cosine_scaled_reward": 1.240987867116928, | |
| "rewards/format_reward": 0.0, | |
| "step": 140 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 715.0223617553711, | |
| "epoch": 1.4359949302915083, | |
| "grad_norm": 0.5516566634178162, | |
| "learning_rate": 2.2221488349019902e-07, | |
| "loss": -0.0049, | |
| "num_tokens": 114492577.0, | |
| "reward": 1.1441184803843498, | |
| "reward_std": 0.4257240351289511, | |
| "rewards/cosine_scaled_reward": 1.1441184356808662, | |
| "rewards/format_reward": 0.0, | |
| "step": 141 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 683.0033798217773, | |
| "epoch": 1.4461343472750317, | |
| "grad_norm": 0.135258749127388, | |
| "learning_rate": 2.1483868153251788e-07, | |
| "loss": 0.0098, | |
| "num_tokens": 115236092.0, | |
| "reward": 1.2539073526859283, | |
| "reward_std": 0.42142050713300705, | |
| "rewards/cosine_scaled_reward": 1.2539073079824448, | |
| "rewards/format_reward": 0.0, | |
| "step": 142 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 783.0201263427734, | |
| "epoch": 1.456273764258555, | |
| "grad_norm": 0.17022480070590973, | |
| "learning_rate": 2.0755333554967346e-07, | |
| "loss": -0.0005, | |
| "num_tokens": 116074878.0, | |
| "reward": 1.0983483716845512, | |
| "reward_std": 0.42650508880615234, | |
| "rewards/cosine_scaled_reward": 1.0983483269810677, | |
| "rewards/format_reward": 0.0, | |
| "step": 143 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 774.2790603637695, | |
| "epoch": 1.4664131812420786, | |
| "grad_norm": 0.1283206343650818, | |
| "learning_rate": 2.0036116674432652e-07, | |
| "loss": -0.0156, | |
| "num_tokens": 116905080.0, | |
| "reward": 1.1725050881505013, | |
| "reward_std": 0.35209081321954727, | |
| "rewards/cosine_scaled_reward": 1.172505035996437, | |
| "rewards/format_reward": 0.0, | |
| "step": 144 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 658.7600746154785, | |
| "epoch": 1.476552598225602, | |
| "grad_norm": 0.14014765620231628, | |
| "learning_rate": 1.9326446663172035e-07, | |
| "loss": 0.0474, | |
| "num_tokens": 117626809.0, | |
| "reward": 1.341113954782486, | |
| "reward_std": 0.4142299462109804, | |
| "rewards/cosine_scaled_reward": 1.3411139100790024, | |
| "rewards/format_reward": 0.0, | |
| "step": 145 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 679.3192367553711, | |
| "epoch": 1.4866920152091254, | |
| "grad_norm": 0.15331801772117615, | |
| "learning_rate": 1.8626549630957395e-07, | |
| "loss": 0.0136, | |
| "num_tokens": 118363623.0, | |
| "reward": 1.19945827126503, | |
| "reward_std": 0.4244145043194294, | |
| "rewards/cosine_scaled_reward": 1.1994582116603851, | |
| "rewards/format_reward": 0.0, | |
| "step": 146 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 752.2890930175781, | |
| "epoch": 1.496831432192649, | |
| "grad_norm": 0.13101238012313843, | |
| "learning_rate": 1.7936648573766954e-07, | |
| "loss": -0.0056, | |
| "num_tokens": 119174386.0, | |
| "reward": 1.2080383449792862, | |
| "reward_std": 0.4013860188424587, | |
| "rewards/cosine_scaled_reward": 1.2080383002758026, | |
| "rewards/format_reward": 0.0, | |
| "step": 147 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 762.8594131469727, | |
| "epoch": 1.5069708491761724, | |
| "grad_norm": 0.12902143597602844, | |
| "learning_rate": 1.725696330273575e-07, | |
| "loss": -0.0383, | |
| "num_tokens": 119988252.0, | |
| "reward": 1.169223740696907, | |
| "reward_std": 0.39561279490590096, | |
| "rewards/cosine_scaled_reward": 1.169223703444004, | |
| "rewards/format_reward": 0.0, | |
| "step": 148 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 700.4531478881836, | |
| "epoch": 1.5171102661596958, | |
| "grad_norm": 0.1390618234872818, | |
| "learning_rate": 1.65877103741212e-07, | |
| "loss": 0.0519, | |
| "num_tokens": 120740586.0, | |
| "reward": 1.37008798122406, | |
| "reward_std": 0.3735492154955864, | |
| "rewards/cosine_scaled_reward": 1.3700879365205765, | |
| "rewards/format_reward": 0.0, | |
| "step": 149 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 706.7366485595703, | |
| "epoch": 1.5272496831432192, | |
| "grad_norm": 0.13693000376224518, | |
| "learning_rate": 1.592910302030544e-07, | |
| "loss": 0.0186, | |
| "num_tokens": 121493126.0, | |
| "reward": 1.1771222800016403, | |
| "reward_std": 0.4693867526948452, | |
| "rewards/cosine_scaled_reward": 1.1771221980452538, | |
| "rewards/format_reward": 0.0, | |
| "step": 150 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 725.279052734375, | |
| "epoch": 1.5373891001267426, | |
| "grad_norm": 0.16313283145427704, | |
| "learning_rate": 1.5281351081856976e-07, | |
| "loss": 0.0023, | |
| "num_tokens": 122274232.0, | |
| "reward": 1.203133448958397, | |
| "reward_std": 0.41216023825109005, | |
| "rewards/cosine_scaled_reward": 1.2031333893537521, | |
| "rewards/format_reward": 0.0, | |
| "step": 151 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 703.8917770385742, | |
| "epoch": 1.5475285171102662, | |
| "grad_norm": 0.16185523569583893, | |
| "learning_rate": 1.4644660940672627e-07, | |
| "loss": -0.0126, | |
| "num_tokens": 123030511.0, | |
| "reward": 1.2789395153522491, | |
| "reward_std": 0.382132101804018, | |
| "rewards/cosine_scaled_reward": 1.2789394706487656, | |
| "rewards/format_reward": 0.0, | |
| "step": 152 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 690.9888610839844, | |
| "epoch": 1.5576679340937896, | |
| "grad_norm": 0.6896323561668396, | |
| "learning_rate": 1.4019235454221856e-07, | |
| "loss": -0.0251, | |
| "num_tokens": 123783709.0, | |
| "reward": 1.230302333831787, | |
| "reward_std": 0.3682190030813217, | |
| "rewards/cosine_scaled_reward": 1.2303023040294647, | |
| "rewards/format_reward": 0.0, | |
| "step": 153 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 702.8527069091797, | |
| "epoch": 1.5678073510773132, | |
| "grad_norm": 0.14129234850406647, | |
| "learning_rate": 1.3405273890913737e-07, | |
| "loss": 0.0005, | |
| "num_tokens": 124535481.0, | |
| "reward": 1.2051593363285065, | |
| "reward_std": 0.4200758896768093, | |
| "rewards/cosine_scaled_reward": 1.205159269273281, | |
| "rewards/format_reward": 0.0, | |
| "step": 154 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 710.2265853881836, | |
| "epoch": 1.5779467680608366, | |
| "grad_norm": 0.1381485015153885, | |
| "learning_rate": 1.280297186660752e-07, | |
| "loss": 0.0181, | |
| "num_tokens": 125290220.0, | |
| "reward": 1.2739061638712883, | |
| "reward_std": 0.4085957519710064, | |
| "rewards/cosine_scaled_reward": 1.2739060744643211, | |
| "rewards/format_reward": 0.0, | |
| "step": 155 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 808.0145492553711, | |
| "epoch": 1.58808618504436, | |
| "grad_norm": 0.13849686086177826, | |
| "learning_rate": 1.2212521282287093e-07, | |
| "loss": -0.005, | |
| "num_tokens": 126134129.0, | |
| "reward": 1.175368033349514, | |
| "reward_std": 0.3809916824102402, | |
| "rewards/cosine_scaled_reward": 1.1753679811954498, | |
| "rewards/format_reward": 0.0, | |
| "step": 156 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 754.8638763427734, | |
| "epoch": 1.5982256020278833, | |
| "grad_norm": 0.14109836518764496, | |
| "learning_rate": 1.1634110262918717e-07, | |
| "loss": -0.0008, | |
| "num_tokens": 126954495.0, | |
| "reward": 1.2158635556697845, | |
| "reward_std": 0.34099204279482365, | |
| "rewards/cosine_scaled_reward": 1.2158635258674622, | |
| "rewards/format_reward": 0.0, | |
| "step": 157 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 775.9598541259766, | |
| "epoch": 1.6083650190114067, | |
| "grad_norm": 0.12850748002529144, | |
| "learning_rate": 1.1067923097512255e-07, | |
| "loss": -0.0005, | |
| "num_tokens": 127780467.0, | |
| "reward": 1.1476298496127129, | |
| "reward_std": 0.4081256799399853, | |
| "rewards/cosine_scaled_reward": 1.1476298123598099, | |
| "rewards/format_reward": 0.0, | |
| "step": 158 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 753.6797256469727, | |
| "epoch": 1.6185044359949303, | |
| "grad_norm": 0.36824139952659607, | |
| "learning_rate": 1.0514140180404202e-07, | |
| "loss": 0.0054, | |
| "num_tokens": 128584820.0, | |
| "reward": 1.0943407788872719, | |
| "reward_std": 0.41071633249521255, | |
| "rewards/cosine_scaled_reward": 1.0943407267332077, | |
| "rewards/format_reward": 0.0, | |
| "step": 159 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 821.4553909301758, | |
| "epoch": 1.6286438529784537, | |
| "grad_norm": 0.15713122487068176, | |
| "learning_rate": 9.972937953781984e-08, | |
| "loss": 0.0185, | |
| "num_tokens": 129467812.0, | |
| "reward": 1.086461715400219, | |
| "reward_std": 0.4641183950006962, | |
| "rewards/cosine_scaled_reward": 1.0864616632461548, | |
| "rewards/format_reward": 0.0, | |
| "step": 160 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 705.2288208007812, | |
| "epoch": 1.6387832699619773, | |
| "grad_norm": 0.13201545178890228, | |
| "learning_rate": 9.444488851467041e-08, | |
| "loss": 0.0098, | |
| "num_tokens": 130217369.0, | |
| "reward": 1.2702905237674713, | |
| "reward_std": 0.3846561126410961, | |
| "rewards/cosine_scaled_reward": 1.2702905088663101, | |
| "rewards/format_reward": 0.0, | |
| "step": 161 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 732.4542770385742, | |
| "epoch": 1.6489226869455007, | |
| "grad_norm": 0.12551482021808624, | |
| "learning_rate": 8.928961243975436e-08, | |
| "loss": 0.0025, | |
| "num_tokens": 131006504.0, | |
| "reward": 1.1392531916499138, | |
| "reward_std": 0.4202544465661049, | |
| "rewards/cosine_scaled_reward": 1.150568701326847, | |
| "rewards/format_reward": 0.0, | |
| "step": 162 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 690.4821739196777, | |
| "epoch": 1.659062103929024, | |
| "grad_norm": 0.14846356213092804, | |
| "learning_rate": 8.426519384872732e-08, | |
| "loss": 0.0151, | |
| "num_tokens": 131747312.0, | |
| "reward": 1.2898453325033188, | |
| "reward_std": 0.3518759645521641, | |
| "rewards/cosine_scaled_reward": 1.2898452877998352, | |
| "rewards/format_reward": 0.0, | |
| "step": 163 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 749.0937881469727, | |
| "epoch": 1.6692015209125475, | |
| "grad_norm": 0.13933837413787842, | |
| "learning_rate": 7.937323358440934e-08, | |
| "loss": 0.028, | |
| "num_tokens": 132549532.0, | |
| "reward": 1.1977392584085464, | |
| "reward_std": 0.43290739692747593, | |
| "rewards/cosine_scaled_reward": 1.1977391764521599, | |
| "rewards/format_reward": 0.0, | |
| "step": 164 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 750.6361999511719, | |
| "epoch": 1.6793409378960709, | |
| "grad_norm": 0.1263691633939743, | |
| "learning_rate": 7.461529028673463e-08, | |
| "loss": 0.0414, | |
| "num_tokens": 133354910.0, | |
| "reward": 1.2933627367019653, | |
| "reward_std": 0.41812827065587044, | |
| "rewards/cosine_scaled_reward": 1.2933626621961594, | |
| "rewards/format_reward": 0.0, | |
| "step": 165 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 748.4531555175781, | |
| "epoch": 1.6894803548795945, | |
| "grad_norm": 0.1337745487689972, | |
| "learning_rate": 6.999287989614971e-08, | |
| "loss": 0.0003, | |
| "num_tokens": 134157964.0, | |
| "reward": 1.1261718794703484, | |
| "reward_std": 0.43879349157214165, | |
| "rewards/cosine_scaled_reward": 1.1261718571186066, | |
| "rewards/format_reward": 0.0, | |
| "step": 166 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 684.155158996582, | |
| "epoch": 1.6996197718631179, | |
| "grad_norm": 0.14675858616828918, | |
| "learning_rate": 6.550747517061656e-08, | |
| "loss": 0.0135, | |
| "num_tokens": 134897511.0, | |
| "reward": 1.31067955493927, | |
| "reward_std": 0.4250149428844452, | |
| "rewards/cosine_scaled_reward": 1.3106794953346252, | |
| "rewards/format_reward": 0.0, | |
| "step": 167 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 708.9955825805664, | |
| "epoch": 1.7097591888466415, | |
| "grad_norm": 0.1361108422279358, | |
| "learning_rate": 6.116050521637218e-08, | |
| "loss": 0.0403, | |
| "num_tokens": 135661099.0, | |
| "reward": 1.2582149505615234, | |
| "reward_std": 0.4354391284286976, | |
| "rewards/cosine_scaled_reward": 1.2582148760557175, | |
| "rewards/format_reward": 0.0, | |
| "step": 168 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 708.2154312133789, | |
| "epoch": 1.7198986058301649, | |
| "grad_norm": 0.13875119388103485, | |
| "learning_rate": 5.6953355032598795e-08, | |
| "loss": -0.0346, | |
| "num_tokens": 136432732.0, | |
| "reward": 1.1267788708209991, | |
| "reward_std": 0.43002479895949364, | |
| "rewards/cosine_scaled_reward": 1.126778818666935, | |
| "rewards/format_reward": 0.0, | |
| "step": 169 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 729.7221374511719, | |
| "epoch": 1.7300380228136882, | |
| "grad_norm": 0.136752188205719, | |
| "learning_rate": 5.288736507014435e-08, | |
| "loss": -0.0081, | |
| "num_tokens": 137210563.0, | |
| "reward": 1.251162275671959, | |
| "reward_std": 0.36924222111701965, | |
| "rewards/cosine_scaled_reward": 1.2511622309684753, | |
| "rewards/format_reward": 0.0, | |
| "step": 170 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 707.0078430175781, | |
| "epoch": 1.7401774397972116, | |
| "grad_norm": 0.132461279630661, | |
| "learning_rate": 4.896383080443933e-08, | |
| "loss": 0.0231, | |
| "num_tokens": 137968794.0, | |
| "reward": 1.2026441097259521, | |
| "reward_std": 0.42860128730535507, | |
| "rewards/cosine_scaled_reward": 1.202644057571888, | |
| "rewards/format_reward": 0.0, | |
| "step": 171 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 690.8460159301758, | |
| "epoch": 1.750316856780735, | |
| "grad_norm": 0.14912405610084534, | |
| "learning_rate": 4.518400232274078e-08, | |
| "loss": 0.0096, | |
| "num_tokens": 138718296.0, | |
| "reward": 1.188114494085312, | |
| "reward_std": 0.39624364115297794, | |
| "rewards/cosine_scaled_reward": 1.2003108784556389, | |
| "rewards/format_reward": 0.0, | |
| "step": 172 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 684.9765930175781, | |
| "epoch": 1.7604562737642584, | |
| "grad_norm": 0.14246663451194763, | |
| "learning_rate": 4.1549083925840165e-08, | |
| "loss": 0.0125, | |
| "num_tokens": 139449483.0, | |
| "reward": 1.2953269332647324, | |
| "reward_std": 0.4603438973426819, | |
| "rewards/cosine_scaled_reward": 1.29532690346241, | |
| "rewards/format_reward": 0.0, | |
| "step": 173 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 685.9721298217773, | |
| "epoch": 1.770595690747782, | |
| "grad_norm": 0.1809549778699875, | |
| "learning_rate": 3.806023374435663e-08, | |
| "loss": -0.0066, | |
| "num_tokens": 140192642.0, | |
| "reward": 1.18511962890625, | |
| "reward_std": 0.365385384298861, | |
| "rewards/cosine_scaled_reward": 1.1851196065545082, | |
| "rewards/format_reward": 0.0, | |
| "step": 174 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 718.2578353881836, | |
| "epoch": 1.7807351077313056, | |
| "grad_norm": 2.9148614406585693, | |
| "learning_rate": 3.4718563369743213e-08, | |
| "loss": 0.0077, | |
| "num_tokens": 140969529.0, | |
| "reward": 1.1897177621722221, | |
| "reward_std": 0.44410305470228195, | |
| "rewards/cosine_scaled_reward": 1.1897177323698997, | |
| "rewards/format_reward": 0.0, | |
| "step": 175 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 672.2578392028809, | |
| "epoch": 1.790874524714829, | |
| "grad_norm": 0.14049698412418365, | |
| "learning_rate": 3.15251375001192e-08, | |
| "loss": 0.0167, | |
| "num_tokens": 141711976.0, | |
| "reward": 1.2624182030558586, | |
| "reward_std": 0.45537084713578224, | |
| "rewards/cosine_scaled_reward": 1.2624181509017944, | |
| "rewards/format_reward": 0.0, | |
| "step": 176 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 697.0145378112793, | |
| "epoch": 1.8010139416983524, | |
| "grad_norm": 0.22702006995677948, | |
| "learning_rate": 2.8480973601043955e-08, | |
| "loss": -0.01, | |
| "num_tokens": 142471309.0, | |
| "reward": 1.249974675476551, | |
| "reward_std": 0.3576275184750557, | |
| "rewards/cosine_scaled_reward": 1.2499746307730675, | |
| "rewards/format_reward": 0.0, | |
| "step": 177 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 744.4810409545898, | |
| "epoch": 1.8111533586818758, | |
| "grad_norm": 0.12813909351825714, | |
| "learning_rate": 2.558704158134023e-08, | |
| "loss": -0.0078, | |
| "num_tokens": 143278388.0, | |
| "reward": 1.1319350376725197, | |
| "reward_std": 0.41413314267992973, | |
| "rewards/cosine_scaled_reward": 1.1319349706172943, | |
| "rewards/format_reward": 0.0, | |
| "step": 178 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 750.3560638427734, | |
| "epoch": 1.8212927756653992, | |
| "grad_norm": 0.13594254851341248, | |
| "learning_rate": 2.2844263484068093e-08, | |
| "loss": 0.0003, | |
| "num_tokens": 144077707.0, | |
| "reward": 1.2112557888031006, | |
| "reward_std": 0.3645152598619461, | |
| "rewards/cosine_scaled_reward": 1.2112557291984558, | |
| "rewards/format_reward": 0.0, | |
| "step": 179 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 682.1417694091797, | |
| "epoch": 1.8314321926489225, | |
| "grad_norm": 0.15148992836475372, | |
| "learning_rate": 2.025351319275137e-08, | |
| "loss": -0.0027, | |
| "num_tokens": 144819874.0, | |
| "reward": 1.2360253632068634, | |
| "reward_std": 0.43914780393242836, | |
| "rewards/cosine_scaled_reward": 1.2360252812504768, | |
| "rewards/format_reward": 0.0, | |
| "step": 180 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 724.106071472168, | |
| "epoch": 1.8415716096324461, | |
| "grad_norm": 0.14325113594532013, | |
| "learning_rate": 1.781561615294652e-08, | |
| "loss": 0.0052, | |
| "num_tokens": 145603137.0, | |
| "reward": 1.226213201880455, | |
| "reward_std": 0.4393893778324127, | |
| "rewards/cosine_scaled_reward": 1.2262131571769714, | |
| "rewards/format_reward": 0.0, | |
| "step": 181 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 782.7399749755859, | |
| "epoch": 1.8517110266159695, | |
| "grad_norm": 0.1272156685590744, | |
| "learning_rate": 1.553134910924636e-08, | |
| "loss": -0.0251, | |
| "num_tokens": 146440576.0, | |
| "reward": 1.0694795548915863, | |
| "reward_std": 0.3888841047883034, | |
| "rewards/cosine_scaled_reward": 1.0694795101881027, | |
| "rewards/format_reward": 0.0, | |
| "step": 182 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 677.8895492553711, | |
| "epoch": 1.8618504435994931, | |
| "grad_norm": 0.14028111100196838, | |
| "learning_rate": 1.340143985779829e-08, | |
| "loss": 0.0087, | |
| "num_tokens": 147174725.0, | |
| "reward": 1.2450221478939056, | |
| "reward_std": 0.3928499836474657, | |
| "rewards/cosine_scaled_reward": 1.2450221329927444, | |
| "rewards/format_reward": 0.0, | |
| "step": 183 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 698.4096298217773, | |
| "epoch": 1.8719898605830165, | |
| "grad_norm": 0.1386420577764511, | |
| "learning_rate": 1.1426567014420297e-08, | |
| "loss": -0.001, | |
| "num_tokens": 147927204.0, | |
| "reward": 1.2862959504127502, | |
| "reward_std": 0.39938454143702984, | |
| "rewards/cosine_scaled_reward": 1.2862958759069443, | |
| "rewards/format_reward": 0.0, | |
| "step": 184 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 749.4877471923828, | |
| "epoch": 1.88212927756654, | |
| "grad_norm": 0.26094314455986023, | |
| "learning_rate": 9.607359798384784e-09, | |
| "loss": 0.0099, | |
| "num_tokens": 148714225.0, | |
| "reward": 1.15062565356493, | |
| "reward_std": 0.49165763705968857, | |
| "rewards/cosine_scaled_reward": 1.150625616312027, | |
| "rewards/format_reward": 0.0, | |
| "step": 185 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 701.1071624755859, | |
| "epoch": 1.8922686945500633, | |
| "grad_norm": 0.14008425176143646, | |
| "learning_rate": 7.944397831941951e-09, | |
| "loss": 0.0034, | |
| "num_tokens": 149488913.0, | |
| "reward": 1.181966707110405, | |
| "reward_std": 0.4252682514488697, | |
| "rewards/cosine_scaled_reward": 1.181966632604599, | |
| "rewards/format_reward": 0.0, | |
| "step": 186 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 779.9766082763672, | |
| "epoch": 1.9024081115335867, | |
| "grad_norm": 0.16207562386989594, | |
| "learning_rate": 6.438210955644452e-09, | |
| "loss": -0.0089, | |
| "num_tokens": 150330196.0, | |
| "reward": 1.0697861537337303, | |
| "reward_std": 0.3918624483048916, | |
| "rewards/cosine_scaled_reward": 1.0697861164808273, | |
| "rewards/format_reward": 0.0, | |
| "step": 187 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 725.6529235839844, | |
| "epoch": 1.9125475285171103, | |
| "grad_norm": 0.1371779590845108, | |
| "learning_rate": 5.0892790595336575e-09, | |
| "loss": 0.0236, | |
| "num_tokens": 151117397.0, | |
| "reward": 1.2266732975840569, | |
| "reward_std": 0.4197930619120598, | |
| "rewards/cosine_scaled_reward": 1.226673237979412, | |
| "rewards/format_reward": 0.0, | |
| "step": 188 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 735.1306228637695, | |
| "epoch": 1.9226869455006337, | |
| "grad_norm": 0.14734813570976257, | |
| "learning_rate": 3.898031930240797e-09, | |
| "loss": -0.013, | |
| "num_tokens": 151901074.0, | |
| "reward": 1.1564877182245255, | |
| "reward_std": 0.4156847130507231, | |
| "rewards/cosine_scaled_reward": 1.1564876809716225, | |
| "rewards/format_reward": 0.0, | |
| "step": 189 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 774.4129943847656, | |
| "epoch": 1.9328263624841573, | |
| "grad_norm": 0.12307488918304443, | |
| "learning_rate": 2.8648491140513264e-09, | |
| "loss": -0.0197, | |
| "num_tokens": 152721396.0, | |
| "reward": 1.2314427196979523, | |
| "reward_std": 0.39738379418849945, | |
| "rewards/cosine_scaled_reward": 1.2314426600933075, | |
| "rewards/format_reward": 0.0, | |
| "step": 190 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 639.6127433776855, | |
| "epoch": 1.9429657794676807, | |
| "grad_norm": 0.15224085748195648, | |
| "learning_rate": 1.9900597959770505e-09, | |
| "loss": 0.0385, | |
| "num_tokens": 153419617.0, | |
| "reward": 1.3687696307897568, | |
| "reward_std": 0.3240409158170223, | |
| "rewards/cosine_scaled_reward": 1.3687695413827896, | |
| "rewards/format_reward": 0.0, | |
| "step": 191 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 727.8538208007812, | |
| "epoch": 1.953105196451204, | |
| "grad_norm": 0.12927082180976868, | |
| "learning_rate": 1.2739426948732424e-09, | |
| "loss": 0.002, | |
| "num_tokens": 154205742.0, | |
| "reward": 1.2560684382915497, | |
| "reward_std": 0.3877686746418476, | |
| "rewards/cosine_scaled_reward": 1.256068378686905, | |
| "rewards/format_reward": 0.0, | |
| "step": 192 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 720.2232513427734, | |
| "epoch": 1.9632446134347274, | |
| "grad_norm": 0.5261313915252686, | |
| "learning_rate": 7.16725974635568e-10, | |
| "loss": 0.0203, | |
| "num_tokens": 154976902.0, | |
| "reward": 1.1852427944540977, | |
| "reward_std": 0.4709174670279026, | |
| "rewards/cosine_scaled_reward": 1.1852427273988724, | |
| "rewards/format_reward": 0.0, | |
| "step": 193 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 744.1283721923828, | |
| "epoch": 1.9733840304182508, | |
| "grad_norm": 0.13184396922588348, | |
| "learning_rate": 3.185871715041255e-10, | |
| "loss": -0.0006, | |
| "num_tokens": 155776489.0, | |
| "reward": 1.153962902724743, | |
| "reward_std": 0.35575347393751144, | |
| "rewards/cosine_scaled_reward": 1.1539628580212593, | |
| "rewards/format_reward": 0.0, | |
| "step": 194 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 681.7768096923828, | |
| "epoch": 1.9835234474017744, | |
| "grad_norm": 0.14284567534923553, | |
| "learning_rate": 7.96531374983589e-11, | |
| "loss": 0.0074, | |
| "num_tokens": 156507841.0, | |
| "reward": 1.263557255268097, | |
| "reward_std": 0.42133441381156445, | |
| "rewards/cosine_scaled_reward": 1.263557218015194, | |
| "rewards/format_reward": 0.0, | |
| "step": 195 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 712.19921875, | |
| "epoch": 1.9936628643852978, | |
| "grad_norm": 0.12404794245958328, | |
| "learning_rate": 0.0, | |
| "loss": -0.0205, | |
| "num_tokens": 157287049.0, | |
| "reward": 1.1777000427246094, | |
| "reward_std": 0.4066210947930813, | |
| "rewards/cosine_scaled_reward": 1.1776999980211258, | |
| "rewards/format_reward": 0.0, | |
| "step": 196 | |
| }, | |
| { | |
| "epoch": 1.9936628643852978, | |
| "step": 196, | |
| "total_flos": 0.0, | |
| "train_loss": -0.009652293185873921, | |
| "train_runtime": 33755.1654, | |
| "train_samples_per_second": 0.654, | |
| "train_steps_per_second": 0.006 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 196, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 2, | |
| "save_steps": 500, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 16, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |