| { | |
| "best_global_step": null, | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.9936628643852978, | |
| "eval_steps": 100, | |
| "global_step": 196, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 544.7745819091797, | |
| "epoch": 0.010139416983523447, | |
| "grad_norm": 0.2261611670255661, | |
| "learning_rate": 5e-08, | |
| "loss": -0.2592, | |
| "num_tokens": 636118.0, | |
| "reward": -0.6724561750888824, | |
| "reward_std": 0.4402390718460083, | |
| "rewards/cosine_scaled_reward": -0.6758043766021729, | |
| "rewards/format_reward": 0.003348214435391128, | |
| "step": 1 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 549.2812843322754, | |
| "epoch": 0.020278833967046894, | |
| "grad_norm": 0.22706830501556396, | |
| "learning_rate": 1e-07, | |
| "loss": -0.2276, | |
| "num_tokens": 1263330.0, | |
| "reward": -0.6239746958017349, | |
| "reward_std": 0.49842649325728416, | |
| "rewards/cosine_scaled_reward": -0.6317871585488319, | |
| "rewards/format_reward": 0.007812500349245965, | |
| "step": 2 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 569.3448944091797, | |
| "epoch": 0.030418250950570342, | |
| "grad_norm": 0.18996624648571014, | |
| "learning_rate": 1.5e-07, | |
| "loss": -0.2165, | |
| "num_tokens": 1913583.0, | |
| "reward": -0.614225223660469, | |
| "reward_std": 0.4692864343523979, | |
| "rewards/cosine_scaled_reward": -0.6231537759304047, | |
| "rewards/format_reward": 0.008928571827709675, | |
| "step": 3 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 539.3951110839844, | |
| "epoch": 0.04055766793409379, | |
| "grad_norm": 0.22352048754692078, | |
| "learning_rate": 2e-07, | |
| "loss": -0.2585, | |
| "num_tokens": 2533833.0, | |
| "reward": -0.6429695598781109, | |
| "reward_std": 0.4445586446672678, | |
| "rewards/cosine_scaled_reward": -0.6507820263504982, | |
| "rewards/format_reward": 0.007812500349245965, | |
| "step": 4 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 576.1462287902832, | |
| "epoch": 0.050697084917617236, | |
| "grad_norm": 0.21681766211986542, | |
| "learning_rate": 2.5e-07, | |
| "loss": -0.2409, | |
| "num_tokens": 3176844.0, | |
| "reward": -0.6119992136955261, | |
| "reward_std": 0.4665570706129074, | |
| "rewards/cosine_scaled_reward": -0.6164634898304939, | |
| "rewards/format_reward": 0.004464285913854837, | |
| "step": 5 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 570.2154197692871, | |
| "epoch": 0.060836501901140684, | |
| "grad_norm": 0.21843321621418, | |
| "learning_rate": 3e-07, | |
| "loss": -0.2408, | |
| "num_tokens": 3816949.0, | |
| "reward": -0.5715240500867367, | |
| "reward_std": 0.5433307699859142, | |
| "rewards/cosine_scaled_reward": -0.5771043859422207, | |
| "rewards/format_reward": 0.005580357392318547, | |
| "step": 6 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 539.3672180175781, | |
| "epoch": 0.07097591888466413, | |
| "grad_norm": 0.21752215921878815, | |
| "learning_rate": 3.5e-07, | |
| "loss": -0.1852, | |
| "num_tokens": 4424646.0, | |
| "reward": -0.5407312288880348, | |
| "reward_std": 0.5957661867141724, | |
| "rewards/cosine_scaled_reward": -0.5518919117748737, | |
| "rewards/format_reward": 0.011160714668221772, | |
| "step": 7 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 558.9989128112793, | |
| "epoch": 0.08111533586818757, | |
| "grad_norm": 0.21230322122573853, | |
| "learning_rate": 4e-07, | |
| "loss": -0.205, | |
| "num_tokens": 5053061.0, | |
| "reward": -0.6614980250597, | |
| "reward_std": 0.47641005739569664, | |
| "rewards/cosine_scaled_reward": -0.667078360915184, | |
| "rewards/format_reward": 0.005580357392318547, | |
| "step": 8 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 576.405158996582, | |
| "epoch": 0.09125475285171103, | |
| "grad_norm": 0.22808963060379028, | |
| "learning_rate": 4.5e-07, | |
| "loss": -0.2588, | |
| "num_tokens": 5692848.0, | |
| "reward": -0.6227121576666832, | |
| "reward_std": 0.45661213248968124, | |
| "rewards/cosine_scaled_reward": -0.6294085532426834, | |
| "rewards/format_reward": 0.006696428870782256, | |
| "step": 9 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 516.9732475280762, | |
| "epoch": 0.10139416983523447, | |
| "grad_norm": 0.2245020717382431, | |
| "learning_rate": 5e-07, | |
| "loss": -0.1928, | |
| "num_tokens": 6282784.0, | |
| "reward": -0.5244979597628117, | |
| "reward_std": 0.5999544225633144, | |
| "rewards/cosine_scaled_reward": -0.5278461389243603, | |
| "rewards/format_reward": 0.003348214435391128, | |
| "step": 10 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 583.3047180175781, | |
| "epoch": 0.11153358681875793, | |
| "grad_norm": 0.22737731039524078, | |
| "learning_rate": 5.5e-07, | |
| "loss": -0.2092, | |
| "num_tokens": 6942961.0, | |
| "reward": -0.609170563519001, | |
| "reward_std": 0.47449312917888165, | |
| "rewards/cosine_scaled_reward": -0.6125187538564205, | |
| "rewards/format_reward": 0.003348214435391128, | |
| "step": 11 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 587.389533996582, | |
| "epoch": 0.12167300380228137, | |
| "grad_norm": 0.22877997159957886, | |
| "learning_rate": 6e-07, | |
| "loss": -0.2051, | |
| "num_tokens": 7600574.0, | |
| "reward": -0.5761966481804848, | |
| "reward_std": 0.47745756432414055, | |
| "rewards/cosine_scaled_reward": -0.5884733907878399, | |
| "rewards/format_reward": 0.012276786146685481, | |
| "step": 12 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 605.9844055175781, | |
| "epoch": 0.13181242078580482, | |
| "grad_norm": 0.20151640474796295, | |
| "learning_rate": 6.5e-07, | |
| "loss": -0.1612, | |
| "num_tokens": 8271720.0, | |
| "reward": -0.5711147747933865, | |
| "reward_std": 0.5248831808567047, | |
| "rewards/cosine_scaled_reward": -0.5789272524416447, | |
| "rewards/format_reward": 0.007812500349245965, | |
| "step": 13 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 554.7109680175781, | |
| "epoch": 0.14195183776932827, | |
| "grad_norm": 0.28496649861335754, | |
| "learning_rate": 7e-07, | |
| "loss": -0.1814, | |
| "num_tokens": 8900917.0, | |
| "reward": -0.5377125255763531, | |
| "reward_std": 0.589600820094347, | |
| "rewards/cosine_scaled_reward": -0.5481268018484116, | |
| "rewards/format_reward": 0.004464285913854837, | |
| "step": 14 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 597.1194534301758, | |
| "epoch": 0.1520912547528517, | |
| "grad_norm": 0.20674261450767517, | |
| "learning_rate": 7.5e-07, | |
| "loss": -0.1812, | |
| "num_tokens": 9564376.0, | |
| "reward": -0.5252576433122158, | |
| "reward_std": 0.5734145380556583, | |
| "rewards/cosine_scaled_reward": -0.5308379791676998, | |
| "rewards/format_reward": 0.005580357392318547, | |
| "step": 15 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 587.7846221923828, | |
| "epoch": 0.16223067173637515, | |
| "grad_norm": 0.18769149482250214, | |
| "learning_rate": 8e-07, | |
| "loss": -0.1947, | |
| "num_tokens": 10217959.0, | |
| "reward": -0.5038141719996929, | |
| "reward_std": 0.6175409816205502, | |
| "rewards/cosine_scaled_reward": -0.5071623660624027, | |
| "rewards/format_reward": 0.003348214435391128, | |
| "step": 16 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 625.3683319091797, | |
| "epoch": 0.17237008871989862, | |
| "grad_norm": 0.3610371947288513, | |
| "learning_rate": 8.499999999999999e-07, | |
| "loss": -0.1733, | |
| "num_tokens": 10912801.0, | |
| "reward": -0.5226632095873356, | |
| "reward_std": 0.5681972987949848, | |
| "rewards/cosine_scaled_reward": -0.5282435454428196, | |
| "rewards/format_reward": 0.005580357392318547, | |
| "step": 17 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 632.5993499755859, | |
| "epoch": 0.18250950570342206, | |
| "grad_norm": 0.183369979262352, | |
| "learning_rate": 9e-07, | |
| "loss": -0.1304, | |
| "num_tokens": 11602954.0, | |
| "reward": -0.4399515353143215, | |
| "reward_std": 0.6363755837082863, | |
| "rewards/cosine_scaled_reward": -0.44553186371922493, | |
| "rewards/format_reward": 0.005580357392318547, | |
| "step": 18 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 607.8593978881836, | |
| "epoch": 0.1926489226869455, | |
| "grad_norm": 0.18773187696933746, | |
| "learning_rate": 9.499999999999999e-07, | |
| "loss": -0.1421, | |
| "num_tokens": 12279580.0, | |
| "reward": -0.31881226040422916, | |
| "reward_std": 0.6840411424636841, | |
| "rewards/cosine_scaled_reward": -0.32439261162653565, | |
| "rewards/format_reward": 0.005580357392318547, | |
| "step": 19 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 647.7310485839844, | |
| "epoch": 0.20278833967046894, | |
| "grad_norm": 0.1795511543750763, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0997, | |
| "num_tokens": 12980371.0, | |
| "reward": -0.29815168026834726, | |
| "reward_std": 0.6471684239804745, | |
| "rewards/cosine_scaled_reward": -0.30038381088525057, | |
| "rewards/format_reward": 0.0022321429569274187, | |
| "step": 20 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 660.1004867553711, | |
| "epoch": 0.21292775665399238, | |
| "grad_norm": 0.1688115894794464, | |
| "learning_rate": 9.999203468625015e-07, | |
| "loss": -0.0604, | |
| "num_tokens": 13693093.0, | |
| "reward": -0.11793683946598321, | |
| "reward_std": 0.6547496430575848, | |
| "rewards/cosine_scaled_reward": -0.11905290477443486, | |
| "rewards/format_reward": 0.0011160714784637094, | |
| "step": 21 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 716.8471298217773, | |
| "epoch": 0.22306717363751585, | |
| "grad_norm": 0.1519731730222702, | |
| "learning_rate": 9.99681412828496e-07, | |
| "loss": -0.0818, | |
| "num_tokens": 14468548.0, | |
| "reward": -0.2080770255997777, | |
| "reward_std": 0.5903773456811905, | |
| "rewards/cosine_scaled_reward": -0.21030915481969714, | |
| "rewards/format_reward": 0.0022321429569274187, | |
| "step": 22 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 675.1094131469727, | |
| "epoch": 0.2332065906210393, | |
| "grad_norm": 0.16422481834888458, | |
| "learning_rate": 9.992832740253644e-07, | |
| "loss": -0.0618, | |
| "num_tokens": 15211694.0, | |
| "reward": -0.13133149535860866, | |
| "reward_std": 0.6078522019088268, | |
| "rewards/cosine_scaled_reward": -0.1324475615983829, | |
| "rewards/format_reward": 0.0011160714784637094, | |
| "step": 23 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 732.0714569091797, | |
| "epoch": 0.24334600760456274, | |
| "grad_norm": 0.14371691644191742, | |
| "learning_rate": 9.987260573051267e-07, | |
| "loss": -0.0698, | |
| "num_tokens": 15997174.0, | |
| "reward": -0.12170885177329183, | |
| "reward_std": 0.5266584008932114, | |
| "rewards/cosine_scaled_reward": -0.12282492127269506, | |
| "rewards/format_reward": 0.0011160714784637094, | |
| "step": 24 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 674.2221298217773, | |
| "epoch": 0.2534854245880862, | |
| "grad_norm": 0.4856448471546173, | |
| "learning_rate": 9.98009940204023e-07, | |
| "loss": -0.047, | |
| "num_tokens": 16732605.0, | |
| "reward": 0.00843854621052742, | |
| "reward_std": 0.524466548115015, | |
| "rewards/cosine_scaled_reward": 0.00843854807317257, | |
| "rewards/format_reward": 0.0, | |
| "step": 25 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 704.0792770385742, | |
| "epoch": 0.26362484157160965, | |
| "grad_norm": 0.15166230499744415, | |
| "learning_rate": 9.971351508859486e-07, | |
| "loss": -0.0499, | |
| "num_tokens": 17482292.0, | |
| "reward": 0.026816920144483447, | |
| "reward_std": 0.5594276264309883, | |
| "rewards/cosine_scaled_reward": 0.02570084948092699, | |
| "rewards/format_reward": 0.0011160714784637094, | |
| "step": 26 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 676.5524826049805, | |
| "epoch": 0.2737642585551331, | |
| "grad_norm": 0.2196025401353836, | |
| "learning_rate": 9.961019680697591e-07, | |
| "loss": -0.0143, | |
| "num_tokens": 18216659.0, | |
| "reward": -0.00026240432634949684, | |
| "reward_std": 0.587770089507103, | |
| "rewards/cosine_scaled_reward": -0.0002624071203172207, | |
| "rewards/format_reward": 0.0, | |
| "step": 27 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 751.2332916259766, | |
| "epoch": 0.28390367553865653, | |
| "grad_norm": 1.717673420906067, | |
| "learning_rate": 9.949107209404663e-07, | |
| "loss": 0.0007, | |
| "num_tokens": 19023404.0, | |
| "reward": 0.06609318475238979, | |
| "reward_std": 0.5609292760491371, | |
| "rewards/cosine_scaled_reward": 0.06609318096889183, | |
| "rewards/format_reward": 0.0, | |
| "step": 28 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 717.3995819091797, | |
| "epoch": 0.29404309252217997, | |
| "grad_norm": 0.1441808044910431, | |
| "learning_rate": 9.935617890443554e-07, | |
| "loss": -0.0205, | |
| "num_tokens": 19806930.0, | |
| "reward": 0.054041724652051926, | |
| "reward_std": 0.5101928748190403, | |
| "rewards/cosine_scaled_reward": 0.05404172372072935, | |
| "rewards/format_reward": 0.0, | |
| "step": 29 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 805.4955749511719, | |
| "epoch": 0.3041825095057034, | |
| "grad_norm": 0.12774203717708588, | |
| "learning_rate": 9.92055602168058e-07, | |
| "loss": -0.007, | |
| "num_tokens": 20674142.0, | |
| "reward": 0.047311737202107906, | |
| "reward_std": 0.4787377752363682, | |
| "rewards/cosine_scaled_reward": 0.04731173254549503, | |
| "rewards/format_reward": 0.0, | |
| "step": 30 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 771.9632110595703, | |
| "epoch": 0.31432192648922685, | |
| "grad_norm": 0.1265403777360916, | |
| "learning_rate": 9.90392640201615e-07, | |
| "loss": 0.0387, | |
| "num_tokens": 21493429.0, | |
| "reward": 0.13890161426388659, | |
| "reward_std": 0.52209198102355, | |
| "rewards/cosine_scaled_reward": 0.1389016114990227, | |
| "rewards/format_reward": 0.0, | |
| "step": 31 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 763.1551666259766, | |
| "epoch": 0.3244613434727503, | |
| "grad_norm": 0.7253857851028442, | |
| "learning_rate": 9.885734329855797e-07, | |
| "loss": 0.023, | |
| "num_tokens": 22296960.0, | |
| "reward": 0.20044966787099838, | |
| "reward_std": 0.4556784965097904, | |
| "rewards/cosine_scaled_reward": 0.20044966042041779, | |
| "rewards/format_reward": 0.0, | |
| "step": 32 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 778.8873138427734, | |
| "epoch": 0.33460076045627374, | |
| "grad_norm": 0.1363135427236557, | |
| "learning_rate": 9.865985601422017e-07, | |
| "loss": -0.0147, | |
| "num_tokens": 23128899.0, | |
| "reward": 0.05959217494819313, | |
| "reward_std": 0.4793633334338665, | |
| "rewards/cosine_scaled_reward": 0.059291763755027205, | |
| "rewards/format_reward": 0.0, | |
| "step": 33 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 773.6607513427734, | |
| "epoch": 0.34474017743979724, | |
| "grad_norm": 0.12697336077690125, | |
| "learning_rate": 9.844686508907537e-07, | |
| "loss": 0.0046, | |
| "num_tokens": 23949587.0, | |
| "reward": 0.15802541188895702, | |
| "reward_std": 0.4121544286608696, | |
| "rewards/cosine_scaled_reward": 0.15802540606819093, | |
| "rewards/format_reward": 0.0, | |
| "step": 34 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 755.8393173217773, | |
| "epoch": 0.3548795944233207, | |
| "grad_norm": 0.13817483186721802, | |
| "learning_rate": 9.821843838470534e-07, | |
| "loss": 0.002, | |
| "num_tokens": 24769547.0, | |
| "reward": 0.15929587977007031, | |
| "reward_std": 0.4162367284297943, | |
| "rewards/cosine_scaled_reward": 0.159295872785151, | |
| "rewards/format_reward": 0.0, | |
| "step": 35 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 758.3839645385742, | |
| "epoch": 0.3650190114068441, | |
| "grad_norm": 0.1282864660024643, | |
| "learning_rate": 9.797464868072486e-07, | |
| "loss": 0.0237, | |
| "num_tokens": 25577171.0, | |
| "reward": 0.193751014303416, | |
| "reward_std": 0.48775994032621384, | |
| "rewards/cosine_scaled_reward": 0.193751011043787, | |
| "rewards/format_reward": 0.0, | |
| "step": 36 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 737.6127624511719, | |
| "epoch": 0.37515842839036756, | |
| "grad_norm": 0.13303731381893158, | |
| "learning_rate": 9.771557365159319e-07, | |
| "loss": 0.0083, | |
| "num_tokens": 26363600.0, | |
| "reward": 0.13768478552810848, | |
| "reward_std": 0.40786123648285866, | |
| "rewards/cosine_scaled_reward": 0.1376847765641287, | |
| "rewards/format_reward": 0.0, | |
| "step": 37 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 753.794677734375, | |
| "epoch": 0.385297845373891, | |
| "grad_norm": 0.137796089053154, | |
| "learning_rate": 9.744129584186597e-07, | |
| "loss": 0.0127, | |
| "num_tokens": 27165608.0, | |
| "reward": 0.12427436339203268, | |
| "reward_std": 0.47414594888687134, | |
| "rewards/cosine_scaled_reward": 0.12427435873541981, | |
| "rewards/format_reward": 0.0, | |
| "step": 38 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 799.005615234375, | |
| "epoch": 0.39543726235741444, | |
| "grad_norm": 0.1327836662530899, | |
| "learning_rate": 9.71519026398956e-07, | |
| "loss": 0.0341, | |
| "num_tokens": 28019301.0, | |
| "reward": 0.12463877908885479, | |
| "reward_std": 0.37975190207362175, | |
| "rewards/cosine_scaled_reward": 0.12463877722620964, | |
| "rewards/format_reward": 0.0, | |
| "step": 39 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 732.2611846923828, | |
| "epoch": 0.4055766793409379, | |
| "grad_norm": 0.14108538627624512, | |
| "learning_rate": 9.68474862499881e-07, | |
| "loss": 0.0519, | |
| "num_tokens": 28794671.0, | |
| "reward": 0.17419127095490694, | |
| "reward_std": 0.44464627653360367, | |
| "rewards/cosine_scaled_reward": 0.17419126071035862, | |
| "rewards/format_reward": 0.0, | |
| "step": 40 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 725.4620819091797, | |
| "epoch": 0.4157160963244613, | |
| "grad_norm": 0.13821274042129517, | |
| "learning_rate": 9.652814366302568e-07, | |
| "loss": 0.0465, | |
| "num_tokens": 29570661.0, | |
| "reward": 0.32581639010459185, | |
| "reward_std": 0.3625021446496248, | |
| "rewards/cosine_scaled_reward": 0.325816385447979, | |
| "rewards/format_reward": 0.0, | |
| "step": 41 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 730.3828430175781, | |
| "epoch": 0.42585551330798477, | |
| "grad_norm": 0.1342957466840744, | |
| "learning_rate": 9.619397662556433e-07, | |
| "loss": 0.0505, | |
| "num_tokens": 30354052.0, | |
| "reward": 0.23176710167899728, | |
| "reward_std": 0.36819201707839966, | |
| "rewards/cosine_scaled_reward": 0.23176709050312638, | |
| "rewards/format_reward": 0.0, | |
| "step": 42 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 795.7176742553711, | |
| "epoch": 0.43599493029150826, | |
| "grad_norm": 0.12134254723787308, | |
| "learning_rate": 9.5845091607416e-07, | |
| "loss": 0.041, | |
| "num_tokens": 31185719.0, | |
| "reward": 0.2366194719215855, | |
| "reward_std": 0.4014102444052696, | |
| "rewards/cosine_scaled_reward": 0.23661946202628314, | |
| "rewards/format_reward": 0.0, | |
| "step": 43 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 801.1931076049805, | |
| "epoch": 0.4461343472750317, | |
| "grad_norm": 0.7807191014289856, | |
| "learning_rate": 9.548159976772592e-07, | |
| "loss": 0.0134, | |
| "num_tokens": 32033636.0, | |
| "reward": 0.21036180562805384, | |
| "reward_std": 0.40362199395895004, | |
| "rewards/cosine_scaled_reward": 0.21036179037764668, | |
| "rewards/format_reward": 0.0, | |
| "step": 44 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 819.1339569091797, | |
| "epoch": 0.45627376425855515, | |
| "grad_norm": 0.12294432520866394, | |
| "learning_rate": 9.510361691955606e-07, | |
| "loss": 0.021, | |
| "num_tokens": 32890028.0, | |
| "reward": 0.12139649732853286, | |
| "reward_std": 0.4755956120789051, | |
| "rewards/cosine_scaled_reward": 0.12139649360324256, | |
| "rewards/format_reward": 0.0, | |
| "step": 45 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 717.7433319091797, | |
| "epoch": 0.4664131812420786, | |
| "grad_norm": 0.3100641071796417, | |
| "learning_rate": 9.471126349298556e-07, | |
| "loss": 0.0361, | |
| "num_tokens": 33663342.0, | |
| "reward": 0.23560622334480286, | |
| "reward_std": 0.42723193392157555, | |
| "rewards/cosine_scaled_reward": 0.23560621310025454, | |
| "rewards/format_reward": 0.0, | |
| "step": 46 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 762.2913284301758, | |
| "epoch": 0.47655259822560203, | |
| "grad_norm": 0.12677805125713348, | |
| "learning_rate": 9.430466449674013e-07, | |
| "loss": -0.0045, | |
| "num_tokens": 34494707.0, | |
| "reward": 0.1484816954471171, | |
| "reward_std": 0.3901702091097832, | |
| "rewards/cosine_scaled_reward": 0.14848168776370585, | |
| "rewards/format_reward": 0.0, | |
| "step": 47 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 789.130615234375, | |
| "epoch": 0.4866920152091255, | |
| "grad_norm": 0.12980540096759796, | |
| "learning_rate": 9.388394947836278e-07, | |
| "loss": -0.008, | |
| "num_tokens": 35328504.0, | |
| "reward": 0.09070475457701832, | |
| "reward_std": 0.3948558494448662, | |
| "rewards/cosine_scaled_reward": 0.09070474898908287, | |
| "rewards/format_reward": 0.0, | |
| "step": 48 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 792.5145416259766, | |
| "epoch": 0.4968314321926489, | |
| "grad_norm": 0.1316457837820053, | |
| "learning_rate": 9.344925248293835e-07, | |
| "loss": 0.0133, | |
| "num_tokens": 36159605.0, | |
| "reward": 0.2209687788854353, | |
| "reward_std": 0.39467399939894676, | |
| "rewards/cosine_scaled_reward": 0.22096876305295154, | |
| "rewards/format_reward": 0.0, | |
| "step": 49 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 776.4721374511719, | |
| "epoch": 0.5069708491761724, | |
| "grad_norm": 0.12961426377296448, | |
| "learning_rate": 9.300071201038501e-07, | |
| "loss": -0.0046, | |
| "num_tokens": 36981500.0, | |
| "reward": 0.15061184321530163, | |
| "reward_std": 0.4538601525127888, | |
| "rewards/cosine_scaled_reward": 0.15061183541547507, | |
| "rewards/format_reward": 0.0, | |
| "step": 50 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 665.3649826049805, | |
| "epoch": 0.5171102661596958, | |
| "grad_norm": 0.1373734474182129, | |
| "learning_rate": 9.253847097132655e-07, | |
| "loss": 0.0202, | |
| "num_tokens": 37708491.0, | |
| "reward": 0.25066345604136586, | |
| "reward_std": 0.41370486095547676, | |
| "rewards/cosine_scaled_reward": 0.25066344952210784, | |
| "rewards/format_reward": 0.0, | |
| "step": 51 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 721.5212249755859, | |
| "epoch": 0.5272496831432193, | |
| "grad_norm": 0.12883824110031128, | |
| "learning_rate": 9.206267664155906e-07, | |
| "loss": 0.0295, | |
| "num_tokens": 38475350.0, | |
| "reward": 0.2094368627294898, | |
| "reward_std": 0.4836310148239136, | |
| "rewards/cosine_scaled_reward": 0.2094368515536189, | |
| "rewards/format_reward": 0.0, | |
| "step": 52 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 748.3538284301758, | |
| "epoch": 0.5373891001267427, | |
| "grad_norm": 0.1301122009754181, | |
| "learning_rate": 9.157348061512726e-07, | |
| "loss": 0.0326, | |
| "num_tokens": 39281955.0, | |
| "reward": 0.19618262629956007, | |
| "reward_std": 0.3902331814169884, | |
| "rewards/cosine_scaled_reward": 0.1961826179176569, | |
| "rewards/format_reward": 0.0, | |
| "step": 53 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 820.4486923217773, | |
| "epoch": 0.5475285171102662, | |
| "grad_norm": 0.12074018269777298, | |
| "learning_rate": 9.107103875602458e-07, | |
| "loss": -0.0192, | |
| "num_tokens": 40135629.0, | |
| "reward": 0.1222395608201623, | |
| "reward_std": 0.4181748628616333, | |
| "rewards/cosine_scaled_reward": 0.12223955010995269, | |
| "rewards/format_reward": 0.0, | |
| "step": 54 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 758.4922256469727, | |
| "epoch": 0.5576679340937896, | |
| "grad_norm": 0.14272169768810272, | |
| "learning_rate": 9.055551114853295e-07, | |
| "loss": 0.0269, | |
| "num_tokens": 40945630.0, | |
| "reward": 0.1248078744392842, | |
| "reward_std": 0.445852842181921, | |
| "rewards/cosine_scaled_reward": 0.12480786850210279, | |
| "rewards/format_reward": 0.0, | |
| "step": 55 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 661.1495895385742, | |
| "epoch": 0.5678073510773131, | |
| "grad_norm": 0.2661519944667816, | |
| "learning_rate": 9.002706204621802e-07, | |
| "loss": 0.0013, | |
| "num_tokens": 41661700.0, | |
| "reward": 0.21717188507318497, | |
| "reward_std": 0.46877430006861687, | |
| "rewards/cosine_scaled_reward": 0.21717186993919313, | |
| "rewards/format_reward": 0.0, | |
| "step": 56 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 708.866096496582, | |
| "epoch": 0.5779467680608364, | |
| "grad_norm": 0.15228039026260376, | |
| "learning_rate": 8.948585981959578e-07, | |
| "loss": 0.0119, | |
| "num_tokens": 42428428.0, | |
| "reward": 0.25196343823336065, | |
| "reward_std": 0.3885682690888643, | |
| "rewards/cosine_scaled_reward": 0.2519634268246591, | |
| "rewards/format_reward": 0.0, | |
| "step": 57 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 707.9698944091797, | |
| "epoch": 0.5880861850443599, | |
| "grad_norm": 0.13880610466003418, | |
| "learning_rate": 8.893207690248775e-07, | |
| "loss": 0.0344, | |
| "num_tokens": 43183833.0, | |
| "reward": 0.22749723307788372, | |
| "reward_std": 0.41020799428224564, | |
| "rewards/cosine_scaled_reward": 0.22749722423031926, | |
| "rewards/format_reward": 0.0, | |
| "step": 58 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 698.7991333007812, | |
| "epoch": 0.5982256020278834, | |
| "grad_norm": 0.14641599357128143, | |
| "learning_rate": 8.836588973708128e-07, | |
| "loss": -0.0323, | |
| "num_tokens": 43951549.0, | |
| "reward": 0.11083738692104816, | |
| "reward_std": 0.45132191106677055, | |
| "rewards/cosine_scaled_reward": 0.11083737714216113, | |
| "rewards/format_reward": 0.0, | |
| "step": 59 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 749.3850860595703, | |
| "epoch": 0.6083650190114068, | |
| "grad_norm": 0.14265136420726776, | |
| "learning_rate": 8.778747871771291e-07, | |
| "loss": 0.0228, | |
| "num_tokens": 44760246.0, | |
| "reward": 0.15983119537122548, | |
| "reward_std": 0.4386202208697796, | |
| "rewards/cosine_scaled_reward": 0.15983118978329003, | |
| "rewards/format_reward": 0.0, | |
| "step": 60 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 700.7500305175781, | |
| "epoch": 0.6185044359949303, | |
| "grad_norm": 0.131936714053154, | |
| "learning_rate": 8.719702813339247e-07, | |
| "loss": 0.0532, | |
| "num_tokens": 45510462.0, | |
| "reward": 0.23273684654850513, | |
| "reward_std": 0.4278288222849369, | |
| "rewards/cosine_scaled_reward": 0.23273683816660196, | |
| "rewards/format_reward": 0.0, | |
| "step": 61 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 729.8102951049805, | |
| "epoch": 0.6286438529784537, | |
| "grad_norm": 0.13723866641521454, | |
| "learning_rate": 8.659472610908627e-07, | |
| "loss": -0.0047, | |
| "num_tokens": 46293764.0, | |
| "reward": 0.15899321692995727, | |
| "reward_std": 0.3875345978885889, | |
| "rewards/cosine_scaled_reward": 0.15899321576580405, | |
| "rewards/format_reward": 0.0, | |
| "step": 62 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 631.8594055175781, | |
| "epoch": 0.6387832699619772, | |
| "grad_norm": 0.15783964097499847, | |
| "learning_rate": 8.598076454577814e-07, | |
| "loss": 0.0349, | |
| "num_tokens": 46984206.0, | |
| "reward": 0.28133094896293187, | |
| "reward_std": 0.4228518716990948, | |
| "rewards/cosine_scaled_reward": 0.2813309368539194, | |
| "rewards/format_reward": 0.0, | |
| "step": 63 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 649.584846496582, | |
| "epoch": 0.6489226869455006, | |
| "grad_norm": 0.15222658216953278, | |
| "learning_rate": 8.535533905932737e-07, | |
| "loss": 0.0307, | |
| "num_tokens": 47714362.0, | |
| "reward": 0.28929764311760664, | |
| "reward_std": 0.3426767401397228, | |
| "rewards/cosine_scaled_reward": 0.2892976254224777, | |
| "rewards/format_reward": 0.0, | |
| "step": 64 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 679.0524826049805, | |
| "epoch": 0.6590621039290241, | |
| "grad_norm": 0.17544390261173248, | |
| "learning_rate": 8.471864891814304e-07, | |
| "loss": 0.0596, | |
| "num_tokens": 48441897.0, | |
| "reward": 0.2452558323275298, | |
| "reward_std": 0.4179961830377579, | |
| "rewards/cosine_scaled_reward": 0.24525582510977983, | |
| "rewards/format_reward": 0.0, | |
| "step": 65 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 734.794677734375, | |
| "epoch": 0.6692015209125475, | |
| "grad_norm": 0.13531185686588287, | |
| "learning_rate": 8.407089697969456e-07, | |
| "loss": 0.0013, | |
| "num_tokens": 49225137.0, | |
| "reward": 0.24510841415030882, | |
| "reward_std": 0.40535635873675346, | |
| "rewards/cosine_scaled_reward": 0.24510839313734323, | |
| "rewards/format_reward": 0.0, | |
| "step": 66 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 763.2321701049805, | |
| "epoch": 0.679340937896071, | |
| "grad_norm": 0.1570795476436615, | |
| "learning_rate": 8.341228962587881e-07, | |
| "loss": 0.0186, | |
| "num_tokens": 50045121.0, | |
| "reward": 0.20169485360383987, | |
| "reward_std": 0.37343652732670307, | |
| "rewards/cosine_scaled_reward": 0.2016948414966464, | |
| "rewards/format_reward": 0.0, | |
| "step": 67 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 628.7924346923828, | |
| "epoch": 0.6894803548795945, | |
| "grad_norm": 0.15324285626411438, | |
| "learning_rate": 8.274303669726426e-07, | |
| "loss": 0.0192, | |
| "num_tokens": 50729839.0, | |
| "reward": 0.32515534153208137, | |
| "reward_std": 0.39401572197675705, | |
| "rewards/cosine_scaled_reward": 0.32515532383695245, | |
| "rewards/format_reward": 0.0, | |
| "step": 68 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 743.4654388427734, | |
| "epoch": 0.6996197718631179, | |
| "grad_norm": 0.14744603633880615, | |
| "learning_rate": 8.206335142623304e-07, | |
| "loss": -0.0119, | |
| "num_tokens": 51525744.0, | |
| "reward": 0.1557501032948494, | |
| "reward_std": 0.4095492400228977, | |
| "rewards/cosine_scaled_reward": 0.15575009072199464, | |
| "rewards/format_reward": 0.0, | |
| "step": 69 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 700.7924575805664, | |
| "epoch": 0.7097591888466414, | |
| "grad_norm": 0.13947679102420807, | |
| "learning_rate": 8.137345036904259e-07, | |
| "loss": 0.0206, | |
| "num_tokens": 52273030.0, | |
| "reward": 0.14582239514857065, | |
| "reward_std": 0.4498204067349434, | |
| "rewards/cosine_scaled_reward": 0.14582238957518712, | |
| "rewards/format_reward": 0.0, | |
| "step": 70 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 674.7065048217773, | |
| "epoch": 0.7198986058301647, | |
| "grad_norm": 0.14676493406295776, | |
| "learning_rate": 8.067355333682797e-07, | |
| "loss": 0.0221, | |
| "num_tokens": 53016335.0, | |
| "reward": 0.18560006003826857, | |
| "reward_std": 0.39886176586151123, | |
| "rewards/cosine_scaled_reward": 0.1856000553816557, | |
| "rewards/format_reward": 0.0, | |
| "step": 71 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 669.2221298217773, | |
| "epoch": 0.7300380228136882, | |
| "grad_norm": 0.13405963778495789, | |
| "learning_rate": 7.996388332556734e-07, | |
| "loss": 0.0196, | |
| "num_tokens": 53744678.0, | |
| "reward": 0.22391042113304138, | |
| "reward_std": 0.40190940350294113, | |
| "rewards/cosine_scaled_reward": 0.2239104132167995, | |
| "rewards/format_reward": 0.0, | |
| "step": 72 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 718.1372909545898, | |
| "epoch": 0.7401774397972116, | |
| "grad_norm": 0.3063336908817291, | |
| "learning_rate": 7.924466644503264e-07, | |
| "loss": -0.0106, | |
| "num_tokens": 54520625.0, | |
| "reward": 0.20024659007322043, | |
| "reward_std": 0.42064137011766434, | |
| "rewards/cosine_scaled_reward": 0.2002465761033818, | |
| "rewards/format_reward": 0.0, | |
| "step": 73 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 641.3616333007812, | |
| "epoch": 0.7503168567807351, | |
| "grad_norm": 0.16075171530246735, | |
| "learning_rate": 7.85161318467482e-07, | |
| "loss": 0.044, | |
| "num_tokens": 55230669.0, | |
| "reward": 0.2528219725936651, | |
| "reward_std": 0.4147798642516136, | |
| "rewards/cosine_scaled_reward": 0.2528219521045685, | |
| "rewards/format_reward": 0.0, | |
| "step": 74 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 631.3672180175781, | |
| "epoch": 0.7604562737642585, | |
| "grad_norm": 0.6628883481025696, | |
| "learning_rate": 7.777851165098011e-07, | |
| "loss": 0.0325, | |
| "num_tokens": 55927054.0, | |
| "reward": 0.3146879682317376, | |
| "reward_std": 0.39327000454068184, | |
| "rewards/cosine_scaled_reward": 0.3146879549603909, | |
| "rewards/format_reward": 0.0, | |
| "step": 75 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 611.7455635070801, | |
| "epoch": 0.770595690747782, | |
| "grad_norm": 0.15209974348545074, | |
| "learning_rate": 7.703204087277988e-07, | |
| "loss": 0.0055, | |
| "num_tokens": 56602418.0, | |
| "reward": 0.135146701708436, | |
| "reward_std": 0.4354245401918888, | |
| "rewards/cosine_scaled_reward": 0.13514669938012958, | |
| "rewards/format_reward": 0.0, | |
| "step": 76 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 662.4486885070801, | |
| "epoch": 0.7807351077313055, | |
| "grad_norm": 0.14389804005622864, | |
| "learning_rate": 7.627695734710564e-07, | |
| "loss": -0.004, | |
| "num_tokens": 57318196.0, | |
| "reward": 0.24701710138469934, | |
| "reward_std": 0.3370856884866953, | |
| "rewards/cosine_scaled_reward": 0.2470170920714736, | |
| "rewards/format_reward": 0.0, | |
| "step": 77 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 684.2477912902832, | |
| "epoch": 0.7908745247148289, | |
| "grad_norm": 0.14715959131717682, | |
| "learning_rate": 7.551350165304499e-07, | |
| "loss": 0.0548, | |
| "num_tokens": 58049058.0, | |
| "reward": 0.30690577276982367, | |
| "reward_std": 0.41454331763088703, | |
| "rewards/cosine_scaled_reward": 0.30690575792687014, | |
| "rewards/format_reward": 0.0, | |
| "step": 78 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 651.327033996582, | |
| "epoch": 0.8010139416983524, | |
| "grad_norm": 0.13967670500278473, | |
| "learning_rate": 7.474191703716338e-07, | |
| "loss": 0.0039, | |
| "num_tokens": 58763207.0, | |
| "reward": 0.24645255785435438, | |
| "reward_std": 0.36919011175632477, | |
| "rewards/cosine_scaled_reward": 0.2464525457471609, | |
| "rewards/format_reward": 0.0, | |
| "step": 79 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 620.6060562133789, | |
| "epoch": 0.8111533586818758, | |
| "grad_norm": 0.1529722511768341, | |
| "learning_rate": 7.396244933600284e-07, | |
| "loss": 0.0101, | |
| "num_tokens": 59443558.0, | |
| "reward": 0.25752069521695375, | |
| "reward_std": 0.4100553132593632, | |
| "rewards/cosine_scaled_reward": 0.25752068776637316, | |
| "rewards/format_reward": 0.0, | |
| "step": 80 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 641.4230117797852, | |
| "epoch": 0.8212927756653993, | |
| "grad_norm": 0.1544528305530548, | |
| "learning_rate": 7.317534689775527e-07, | |
| "loss": 0.027, | |
| "num_tokens": 60137977.0, | |
| "reward": 0.22499833535403013, | |
| "reward_std": 0.406455984339118, | |
| "rewards/cosine_scaled_reward": 0.22499832697212696, | |
| "rewards/format_reward": 0.0, | |
| "step": 81 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 616.1852912902832, | |
| "epoch": 0.8314321926489227, | |
| "grad_norm": 0.20284396409988403, | |
| "learning_rate": 7.238086050313562e-07, | |
| "loss": -0.0023, | |
| "num_tokens": 60825335.0, | |
| "reward": 0.1846146322786808, | |
| "reward_std": 0.42002810910344124, | |
| "rewards/cosine_scaled_reward": 0.1846146178431809, | |
| "rewards/format_reward": 0.0, | |
| "step": 82 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 635.2466812133789, | |
| "epoch": 0.8415716096324461, | |
| "grad_norm": 0.163020059466362, | |
| "learning_rate": 7.157924328548002e-07, | |
| "loss": 0.0196, | |
| "num_tokens": 61528108.0, | |
| "reward": 0.2309768390841782, | |
| "reward_std": 0.4035301711410284, | |
| "rewards/cosine_scaled_reward": 0.2309768325649202, | |
| "rewards/format_reward": 0.0, | |
| "step": 83 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 632.155158996582, | |
| "epoch": 0.8517110266159695, | |
| "grad_norm": 0.15208573639392853, | |
| "learning_rate": 7.077075065009433e-07, | |
| "loss": 0.0356, | |
| "num_tokens": 62227831.0, | |
| "reward": 0.29583334969356656, | |
| "reward_std": 0.4564223028719425, | |
| "rewards/cosine_scaled_reward": 0.29583333851769567, | |
| "rewards/format_reward": 0.0, | |
| "step": 84 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 626.7265930175781, | |
| "epoch": 0.861850443599493, | |
| "grad_norm": 0.15748122334480286, | |
| "learning_rate": 6.995564019287869e-07, | |
| "loss": 0.0, | |
| "num_tokens": 62919194.0, | |
| "reward": 0.16931751649826765, | |
| "reward_std": 0.3771196622401476, | |
| "rewards/cosine_scaled_reward": 0.1693175071850419, | |
| "rewards/format_reward": 0.0, | |
| "step": 85 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 665.608283996582, | |
| "epoch": 0.8719898605830165, | |
| "grad_norm": 0.18250492215156555, | |
| "learning_rate": 6.913417161825449e-07, | |
| "loss": -0.0308, | |
| "num_tokens": 63650491.0, | |
| "reward": 0.20493324054405093, | |
| "reward_std": 0.3730998281389475, | |
| "rewards/cosine_scaled_reward": 0.2049332265742123, | |
| "rewards/format_reward": 0.0, | |
| "step": 86 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 622.6272583007812, | |
| "epoch": 0.8821292775665399, | |
| "grad_norm": 0.15512950718402863, | |
| "learning_rate": 6.830660665641897e-07, | |
| "loss": 0.0071, | |
| "num_tokens": 64338085.0, | |
| "reward": 0.2438387837028131, | |
| "reward_std": 0.40685577876865864, | |
| "rewards/cosine_scaled_reward": 0.24383877718355507, | |
| "rewards/format_reward": 0.0, | |
| "step": 87 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 664.6975784301758, | |
| "epoch": 0.8922686945500634, | |
| "grad_norm": 0.9565799832344055, | |
| "learning_rate": 6.747320897995492e-07, | |
| "loss": 0.0461, | |
| "num_tokens": 65062646.0, | |
| "reward": 0.25703565403819084, | |
| "reward_std": 0.4041633568704128, | |
| "rewards/cosine_scaled_reward": 0.2570356400683522, | |
| "rewards/format_reward": 0.0, | |
| "step": 88 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 588.4866333007812, | |
| "epoch": 0.9024081115335868, | |
| "grad_norm": 0.15928997099399567, | |
| "learning_rate": 6.66342441198212e-07, | |
| "loss": 0.003, | |
| "num_tokens": 65713834.0, | |
| "reward": 0.22450890811160207, | |
| "reward_std": 0.3792395293712616, | |
| "rewards/cosine_scaled_reward": 0.22450890205800533, | |
| "rewards/format_reward": 0.0, | |
| "step": 89 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 617.6417694091797, | |
| "epoch": 0.9125475285171103, | |
| "grad_norm": 0.16051417589187622, | |
| "learning_rate": 6.578997938075125e-07, | |
| "loss": 0.0237, | |
| "num_tokens": 66391713.0, | |
| "reward": 0.17285537254065275, | |
| "reward_std": 0.4654933251440525, | |
| "rewards/cosine_scaled_reward": 0.17285536974668503, | |
| "rewards/format_reward": 0.0, | |
| "step": 90 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 612.7511444091797, | |
| "epoch": 0.9226869455006337, | |
| "grad_norm": 0.1583762913942337, | |
| "learning_rate": 6.494068375608646e-07, | |
| "loss": -0.022, | |
| "num_tokens": 67075362.0, | |
| "reward": 0.2817280013114214, | |
| "reward_std": 0.3609468787908554, | |
| "rewards/cosine_scaled_reward": 0.28172798082232475, | |
| "rewards/format_reward": 0.0, | |
| "step": 91 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 647.9185485839844, | |
| "epoch": 0.9328263624841572, | |
| "grad_norm": 0.16609789431095123, | |
| "learning_rate": 6.408662784207149e-07, | |
| "loss": 0.0336, | |
| "num_tokens": 67795273.0, | |
| "reward": 0.2746627312153578, | |
| "reward_std": 0.41118355095386505, | |
| "rewards/cosine_scaled_reward": 0.2746627163141966, | |
| "rewards/format_reward": 0.0, | |
| "step": 92 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 586.2076110839844, | |
| "epoch": 0.9429657794676806, | |
| "grad_norm": 0.17962822318077087, | |
| "learning_rate": 6.322808375163895e-07, | |
| "loss": 0.0304, | |
| "num_tokens": 68447395.0, | |
| "reward": 0.2662193002179265, | |
| "reward_std": 0.4135792311280966, | |
| "rewards/cosine_scaled_reward": 0.2662192853167653, | |
| "rewards/format_reward": 0.0, | |
| "step": 93 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 628.2835159301758, | |
| "epoch": 0.9531051964512041, | |
| "grad_norm": 0.15532483160495758, | |
| "learning_rate": 6.236532502771077e-07, | |
| "loss": -0.0041, | |
| "num_tokens": 69143937.0, | |
| "reward": 0.22241216152906418, | |
| "reward_std": 0.41293996572494507, | |
| "rewards/cosine_scaled_reward": 0.22241215035319328, | |
| "rewards/format_reward": 0.0, | |
| "step": 94 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 581.4933280944824, | |
| "epoch": 0.9632446134347274, | |
| "grad_norm": 0.15709497034549713, | |
| "learning_rate": 6.149862655604403e-07, | |
| "loss": 0.0438, | |
| "num_tokens": 69790643.0, | |
| "reward": 0.2721885642968118, | |
| "reward_std": 0.40717606246471405, | |
| "rewards/cosine_scaled_reward": 0.2721885568462312, | |
| "rewards/format_reward": 0.0, | |
| "step": 95 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 614.9408721923828, | |
| "epoch": 0.973384030418251, | |
| "grad_norm": 0.16992299258708954, | |
| "learning_rate": 6.062826447764883e-07, | |
| "loss": 0.037, | |
| "num_tokens": 70464374.0, | |
| "reward": 0.2846832238137722, | |
| "reward_std": 0.3609850201755762, | |
| "rewards/cosine_scaled_reward": 0.2846832163631916, | |
| "rewards/format_reward": 0.0, | |
| "step": 96 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 633.0982398986816, | |
| "epoch": 0.9835234474017744, | |
| "grad_norm": 0.14964738488197327, | |
| "learning_rate": 5.975451610080642e-07, | |
| "loss": 0.0147, | |
| "num_tokens": 71159438.0, | |
| "reward": 0.19606254808604717, | |
| "reward_std": 0.3826338779181242, | |
| "rewards/cosine_scaled_reward": 0.19606254203245044, | |
| "rewards/format_reward": 0.0, | |
| "step": 97 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 603.615234375, | |
| "epoch": 0.9936628643852978, | |
| "grad_norm": 0.16081441938877106, | |
| "learning_rate": 5.887765981271517e-07, | |
| "loss": 0.0428, | |
| "num_tokens": 71852378.0, | |
| "reward": 0.2125524217262864, | |
| "reward_std": 0.3838059641420841, | |
| "rewards/cosine_scaled_reward": 0.21255241055041552, | |
| "rewards/format_reward": 0.0, | |
| "step": 98 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 590.7076225280762, | |
| "epoch": 1.0101394169835234, | |
| "grad_norm": 0.1603391319513321, | |
| "learning_rate": 5.7997974990793e-07, | |
| "loss": 0.0057, | |
| "num_tokens": 72510644.0, | |
| "reward": 0.2637674935394898, | |
| "reward_std": 0.4375726915895939, | |
| "rewards/cosine_scaled_reward": 0.26376748620532453, | |
| "rewards/format_reward": 0.0, | |
| "step": 99 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 611.1406593322754, | |
| "epoch": 1.020278833967047, | |
| "grad_norm": 0.16759544610977173, | |
| "learning_rate": 5.711574191366427e-07, | |
| "loss": 0.0233, | |
| "num_tokens": 73185602.0, | |
| "reward": 0.24167153984308243, | |
| "reward_std": 0.3558240346610546, | |
| "rewards/cosine_scaled_reward": 0.2416715300641954, | |
| "rewards/format_reward": 0.0, | |
| "step": 100 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 630.9598426818848, | |
| "epoch": 1.0304182509505704, | |
| "grad_norm": 0.16307219862937927, | |
| "learning_rate": 5.623124167185929e-07, | |
| "loss": 0.0067, | |
| "num_tokens": 73873470.0, | |
| "reward": 0.26334861340001225, | |
| "reward_std": 0.3806659858673811, | |
| "rewards/cosine_scaled_reward": 0.26334861130453646, | |
| "rewards/format_reward": 0.0, | |
| "step": 101 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 563.8035888671875, | |
| "epoch": 1.0405576679340938, | |
| "grad_norm": 0.18253548443317413, | |
| "learning_rate": 5.534475607825565e-07, | |
| "loss": 0.0047, | |
| "num_tokens": 74505926.0, | |
| "reward": 0.24372864328324795, | |
| "reward_std": 0.3952293395996094, | |
| "rewards/cosine_scaled_reward": 0.2437286302447319, | |
| "rewards/format_reward": 0.0, | |
| "step": 102 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 634.3750267028809, | |
| "epoch": 1.0506970849176172, | |
| "grad_norm": 0.16632695496082306, | |
| "learning_rate": 5.445656757828879e-07, | |
| "loss": -0.0255, | |
| "num_tokens": 75205630.0, | |
| "reward": 0.07845631497912109, | |
| "reward_std": 0.38748307153582573, | |
| "rewards/cosine_scaled_reward": 0.07845630997326225, | |
| "rewards/format_reward": 0.0, | |
| "step": 103 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 601.0971221923828, | |
| "epoch": 1.0608365019011408, | |
| "grad_norm": 0.14292435348033905, | |
| "learning_rate": 5.356695915996161e-07, | |
| "loss": -0.0234, | |
| "num_tokens": 75878573.0, | |
| "reward": 0.1749452566727996, | |
| "reward_std": 0.44058991968631744, | |
| "rewards/cosine_scaled_reward": 0.17494524829089642, | |
| "rewards/format_reward": 0.0, | |
| "step": 104 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 558.7801628112793, | |
| "epoch": 1.0709759188846641, | |
| "grad_norm": 0.16877137124538422, | |
| "learning_rate": 5.267621426368075e-07, | |
| "loss": 0.0282, | |
| "num_tokens": 76498456.0, | |
| "reward": 0.29683027416467667, | |
| "reward_std": 0.43279460445046425, | |
| "rewards/cosine_scaled_reward": 0.2968302555382252, | |
| "rewards/format_reward": 0.0, | |
| "step": 105 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 506.5335006713867, | |
| "epoch": 1.0811153358681875, | |
| "grad_norm": 0.17591001093387604, | |
| "learning_rate": 5.178461669194903e-07, | |
| "loss": 0.0483, | |
| "num_tokens": 77079518.0, | |
| "reward": 0.3861896749585867, | |
| "reward_std": 0.3774683568626642, | |
| "rewards/cosine_scaled_reward": 0.3861896488815546, | |
| "rewards/format_reward": 0.0, | |
| "step": 106 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 534.6607398986816, | |
| "epoch": 1.091254752851711, | |
| "grad_norm": 0.15835636854171753, | |
| "learning_rate": 5.08924505189423e-07, | |
| "loss": -0.0117, | |
| "num_tokens": 77684534.0, | |
| "reward": 0.28510209172964096, | |
| "reward_std": 0.31983859837055206, | |
| "rewards/cosine_scaled_reward": 0.28510207310318947, | |
| "rewards/format_reward": 0.0, | |
| "step": 107 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 594.6439895629883, | |
| "epoch": 1.1013941698352345, | |
| "grad_norm": 0.20408962666988373, | |
| "learning_rate": 5e-07, | |
| "loss": 0.0081, | |
| "num_tokens": 78341455.0, | |
| "reward": 0.2976014781743288, | |
| "reward_std": 0.3984498456120491, | |
| "rewards/cosine_scaled_reward": 0.29760145768523216, | |
| "rewards/format_reward": 0.0, | |
| "step": 108 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 630.444221496582, | |
| "epoch": 1.111533586818758, | |
| "grad_norm": 0.1533069759607315, | |
| "learning_rate": 4.91075494810577e-07, | |
| "loss": 0.0265, | |
| "num_tokens": 79024325.0, | |
| "reward": 0.25680719036608934, | |
| "reward_std": 0.4040773529559374, | |
| "rewards/cosine_scaled_reward": 0.25680717546492815, | |
| "rewards/format_reward": 0.0, | |
| "step": 109 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 637.928596496582, | |
| "epoch": 1.1216730038022813, | |
| "grad_norm": 0.16302607953548431, | |
| "learning_rate": 4.821538330805098e-07, | |
| "loss": -0.0033, | |
| "num_tokens": 79722429.0, | |
| "reward": 0.2886841148138046, | |
| "reward_std": 0.3645545728504658, | |
| "rewards/cosine_scaled_reward": 0.28868410736322403, | |
| "rewards/format_reward": 0.0, | |
| "step": 110 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 614.0390968322754, | |
| "epoch": 1.131812420785805, | |
| "grad_norm": 0.15493924915790558, | |
| "learning_rate": 4.732378573631924e-07, | |
| "loss": 0.0113, | |
| "num_tokens": 80395336.0, | |
| "reward": 0.25373244076035917, | |
| "reward_std": 0.3782533872872591, | |
| "rewards/cosine_scaled_reward": 0.2537324334261939, | |
| "rewards/format_reward": 0.0, | |
| "step": 111 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 615.8136520385742, | |
| "epoch": 1.1419518377693283, | |
| "grad_norm": 1.329131841659546, | |
| "learning_rate": 4.643304084003838e-07, | |
| "loss": 0.028, | |
| "num_tokens": 81074553.0, | |
| "reward": 0.19473237730562687, | |
| "reward_std": 0.4509877860546112, | |
| "rewards/cosine_scaled_reward": 0.19473236822523177, | |
| "rewards/format_reward": 0.0, | |
| "step": 112 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 605.2098503112793, | |
| "epoch": 1.1520912547528517, | |
| "grad_norm": 0.16996121406555176, | |
| "learning_rate": 4.55434324217112e-07, | |
| "loss": 0.0026, | |
| "num_tokens": 81746061.0, | |
| "reward": 0.1302806935273111, | |
| "reward_std": 0.38874428533017635, | |
| "rewards/cosine_scaled_reward": 0.13028068700805306, | |
| "rewards/format_reward": 0.0, | |
| "step": 113 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 596.4888648986816, | |
| "epoch": 1.162230671736375, | |
| "grad_norm": 0.1666853278875351, | |
| "learning_rate": 4.4655243921744367e-07, | |
| "loss": -0.0409, | |
| "num_tokens": 82405475.0, | |
| "reward": 0.21928726602345705, | |
| "reward_std": 0.3403788786381483, | |
| "rewards/cosine_scaled_reward": 0.21928725019097328, | |
| "rewards/format_reward": 0.0, | |
| "step": 114 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 564.8895378112793, | |
| "epoch": 1.1723700887198987, | |
| "grad_norm": 0.16143201291561127, | |
| "learning_rate": 4.37687583281407e-07, | |
| "loss": -0.0382, | |
| "num_tokens": 83038832.0, | |
| "reward": 0.21677181296399795, | |
| "reward_std": 0.35355726443231106, | |
| "rewards/cosine_scaled_reward": 0.21677180923870765, | |
| "rewards/format_reward": 0.0, | |
| "step": 115 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 553.4587287902832, | |
| "epoch": 1.182509505703422, | |
| "grad_norm": 0.16877496242523193, | |
| "learning_rate": 4.2884258086335745e-07, | |
| "loss": 0.0313, | |
| "num_tokens": 83673971.0, | |
| "reward": 0.2728145425207913, | |
| "reward_std": 0.42727693170309067, | |
| "rewards/cosine_scaled_reward": 0.2728145350702107, | |
| "rewards/format_reward": 0.0, | |
| "step": 116 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 550.5703315734863, | |
| "epoch": 1.1926489226869454, | |
| "grad_norm": 0.17830805480480194, | |
| "learning_rate": 4.2002025009206987e-07, | |
| "loss": 0.0008, | |
| "num_tokens": 84299234.0, | |
| "reward": 0.1771111206180649, | |
| "reward_std": 0.42238056659698486, | |
| "rewards/cosine_scaled_reward": 0.17711111830431037, | |
| "rewards/format_reward": 0.0, | |
| "step": 117 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 588.3515815734863, | |
| "epoch": 1.202788339670469, | |
| "grad_norm": 0.17095865309238434, | |
| "learning_rate": 4.1122340187284845e-07, | |
| "loss": 0.0134, | |
| "num_tokens": 84948685.0, | |
| "reward": 0.20526447892189026, | |
| "reward_std": 0.43867457658052444, | |
| "rewards/cosine_scaled_reward": 0.20526447147130966, | |
| "rewards/format_reward": 0.0, | |
| "step": 118 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 536.7544975280762, | |
| "epoch": 1.2129277566539924, | |
| "grad_norm": 0.1749478578567505, | |
| "learning_rate": 4.0245483899193586e-07, | |
| "loss": 0.0174, | |
| "num_tokens": 85556993.0, | |
| "reward": 0.24879271956160665, | |
| "reward_std": 0.38236525282263756, | |
| "rewards/cosine_scaled_reward": 0.24879270186647773, | |
| "rewards/format_reward": 0.0, | |
| "step": 119 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 558.5870742797852, | |
| "epoch": 1.2230671736375158, | |
| "grad_norm": 1.2731257677078247, | |
| "learning_rate": 3.937173552235116e-07, | |
| "loss": -0.0515, | |
| "num_tokens": 86190295.0, | |
| "reward": 0.1734993983991444, | |
| "reward_std": 0.3362387754023075, | |
| "rewards/cosine_scaled_reward": 0.1734993918798864, | |
| "rewards/format_reward": 0.0, | |
| "step": 120 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 564.9698867797852, | |
| "epoch": 1.2332065906210392, | |
| "grad_norm": 0.20518043637275696, | |
| "learning_rate": 3.850137344395598e-07, | |
| "loss": 0.008, | |
| "num_tokens": 86819724.0, | |
| "reward": 0.2829995099455118, | |
| "reward_std": 0.33430546894669533, | |
| "rewards/cosine_scaled_reward": 0.2829994931817055, | |
| "rewards/format_reward": 0.0, | |
| "step": 121 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 628.7623138427734, | |
| "epoch": 1.2433460076045628, | |
| "grad_norm": 0.1478431224822998, | |
| "learning_rate": 3.763467497228922e-07, | |
| "loss": -0.0359, | |
| "num_tokens": 87521127.0, | |
| "reward": 0.16334752831608057, | |
| "reward_std": 0.3575255237519741, | |
| "rewards/cosine_scaled_reward": 0.16334752179682255, | |
| "rewards/format_reward": 0.0, | |
| "step": 122 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 576.7277030944824, | |
| "epoch": 1.2534854245880862, | |
| "grad_norm": 0.20034752786159515, | |
| "learning_rate": 3.677191624836106e-07, | |
| "loss": 0.0205, | |
| "num_tokens": 88159587.0, | |
| "reward": 0.32477190531790257, | |
| "reward_std": 0.3922874592244625, | |
| "rewards/cosine_scaled_reward": 0.3247718848288059, | |
| "rewards/format_reward": 0.0, | |
| "step": 123 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 611.7667617797852, | |
| "epoch": 1.2636248415716096, | |
| "grad_norm": 0.1693291813135147, | |
| "learning_rate": 3.591337215792851e-07, | |
| "loss": 0.0109, | |
| "num_tokens": 88835898.0, | |
| "reward": 0.23217581922654063, | |
| "reward_std": 0.34129480831325054, | |
| "rewards/cosine_scaled_reward": 0.23217580071650445, | |
| "rewards/format_reward": 0.0, | |
| "step": 124 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 557.9799385070801, | |
| "epoch": 1.2737642585551332, | |
| "grad_norm": 0.1676970273256302, | |
| "learning_rate": 3.505931624391355e-07, | |
| "loss": 0.0225, | |
| "num_tokens": 89454112.0, | |
| "reward": 0.20786370153655298, | |
| "reward_std": 0.4651025366038084, | |
| "rewards/cosine_scaled_reward": 0.20786369033157825, | |
| "rewards/format_reward": 0.0, | |
| "step": 125 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 545.6931037902832, | |
| "epoch": 1.2839036755386566, | |
| "grad_norm": 0.1678163707256317, | |
| "learning_rate": 3.421002061924876e-07, | |
| "loss": 0.0151, | |
| "num_tokens": 90069917.0, | |
| "reward": 0.2275780588388443, | |
| "reward_std": 0.4175031315535307, | |
| "rewards/cosine_scaled_reward": 0.22757804207503796, | |
| "rewards/format_reward": 0.0, | |
| "step": 126 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 540.5000152587891, | |
| "epoch": 1.29404309252218, | |
| "grad_norm": 0.17660397291183472, | |
| "learning_rate": 3.3365755880178807e-07, | |
| "loss": 0.0301, | |
| "num_tokens": 90675909.0, | |
| "reward": 0.2635859549045563, | |
| "reward_std": 0.41772597655653954, | |
| "rewards/cosine_scaled_reward": 0.2635859474539757, | |
| "rewards/format_reward": 0.0, | |
| "step": 127 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 594.244441986084, | |
| "epoch": 1.3041825095057034, | |
| "grad_norm": 0.17012734711170197, | |
| "learning_rate": 3.2526791020045087e-07, | |
| "loss": 0.0215, | |
| "num_tokens": 91340952.0, | |
| "reward": 0.149813212454319, | |
| "reward_std": 0.4470374099910259, | |
| "rewards/cosine_scaled_reward": 0.14981320640072227, | |
| "rewards/format_reward": 0.0, | |
| "step": 128 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 542.3348503112793, | |
| "epoch": 1.3143219264892267, | |
| "grad_norm": 0.174094557762146, | |
| "learning_rate": 3.169339334358104e-07, | |
| "loss": 0.0523, | |
| "num_tokens": 91960228.0, | |
| "reward": 0.26255749678239226, | |
| "reward_std": 0.32982037030160427, | |
| "rewards/cosine_scaled_reward": 0.26255749305710196, | |
| "rewards/format_reward": 0.0, | |
| "step": 129 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 567.3292694091797, | |
| "epoch": 1.3244613434727504, | |
| "grad_norm": 0.16861200332641602, | |
| "learning_rate": 3.086582838174551e-07, | |
| "loss": -0.0835, | |
| "num_tokens": 92601043.0, | |
| "reward": 0.1493756880190631, | |
| "reward_std": 0.3699100259691477, | |
| "rewards/cosine_scaled_reward": 0.1493756801019117, | |
| "rewards/format_reward": 0.0, | |
| "step": 130 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 578.1283683776855, | |
| "epoch": 1.3346007604562737, | |
| "grad_norm": 1.8984020948410034, | |
| "learning_rate": 3.004435980712129e-07, | |
| "loss": -0.0221, | |
| "num_tokens": 93242510.0, | |
| "reward": 0.23330406891182065, | |
| "reward_std": 0.4134122505784035, | |
| "rewards/cosine_scaled_reward": 0.23330405401065946, | |
| "rewards/format_reward": 0.0, | |
| "step": 131 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 538.444221496582, | |
| "epoch": 1.3447401774397973, | |
| "grad_norm": 0.1705993413925171, | |
| "learning_rate": 2.922924934990568e-07, | |
| "loss": 0.0105, | |
| "num_tokens": 93850108.0, | |
| "reward": 0.18292724061757326, | |
| "reward_std": 0.4480227939784527, | |
| "rewards/cosine_scaled_reward": 0.18292723130434752, | |
| "rewards/format_reward": 0.0, | |
| "step": 132 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 524.5457763671875, | |
| "epoch": 1.3548795944233207, | |
| "grad_norm": 0.20817777514457703, | |
| "learning_rate": 2.8420756714519954e-07, | |
| "loss": 0.0035, | |
| "num_tokens": 94464405.0, | |
| "reward": 0.2242459380067885, | |
| "reward_std": 0.3740137480199337, | |
| "rewards/cosine_scaled_reward": 0.2242459305562079, | |
| "rewards/format_reward": 0.0, | |
| "step": 133 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 542.7221260070801, | |
| "epoch": 1.3650190114068441, | |
| "grad_norm": 0.1813177764415741, | |
| "learning_rate": 2.7619139496864376e-07, | |
| "loss": -0.029, | |
| "num_tokens": 95086668.0, | |
| "reward": 0.13100900989957154, | |
| "reward_std": 0.3776312284171581, | |
| "rewards/cosine_scaled_reward": 0.13100900826975703, | |
| "rewards/format_reward": 0.0, | |
| "step": 134 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 500.7243461608887, | |
| "epoch": 1.3751584283903675, | |
| "grad_norm": 0.18803931772708893, | |
| "learning_rate": 2.6824653102244727e-07, | |
| "loss": 0.0016, | |
| "num_tokens": 95669389.0, | |
| "reward": 0.189885966014117, | |
| "reward_std": 0.45219165831804276, | |
| "rewards/cosine_scaled_reward": 0.18988595390692353, | |
| "rewards/format_reward": 0.0, | |
| "step": 135 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 512.5524864196777, | |
| "epoch": 1.385297845373891, | |
| "grad_norm": 0.21896012127399445, | |
| "learning_rate": 2.603755066399718e-07, | |
| "loss": -0.0237, | |
| "num_tokens": 96258716.0, | |
| "reward": 0.17706315219402313, | |
| "reward_std": 0.4132867194712162, | |
| "rewards/cosine_scaled_reward": 0.1770631456747651, | |
| "rewards/format_reward": 0.0, | |
| "step": 136 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 611.787971496582, | |
| "epoch": 1.3954372623574145, | |
| "grad_norm": 0.16569751501083374, | |
| "learning_rate": 2.5258082962836614e-07, | |
| "loss": -0.0733, | |
| "num_tokens": 96941438.0, | |
| "reward": 0.07232440868392587, | |
| "reward_std": 0.3806895762681961, | |
| "rewards/cosine_scaled_reward": 0.07232440612278879, | |
| "rewards/format_reward": 0.0, | |
| "step": 137 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 532.1986846923828, | |
| "epoch": 1.4055766793409379, | |
| "grad_norm": 0.18526431918144226, | |
| "learning_rate": 2.4486498346955023e-07, | |
| "loss": 0.0031, | |
| "num_tokens": 97541120.0, | |
| "reward": 0.18689691158942878, | |
| "reward_std": 0.4354799725115299, | |
| "rewards/cosine_scaled_reward": 0.18689690087921917, | |
| "rewards/format_reward": 0.0, | |
| "step": 138 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 538.1942138671875, | |
| "epoch": 1.4157160963244613, | |
| "grad_norm": 0.1791687160730362, | |
| "learning_rate": 2.372304265289436e-07, | |
| "loss": -0.0459, | |
| "num_tokens": 98147638.0, | |
| "reward": 0.13275248603895307, | |
| "reward_std": 0.44364722445607185, | |
| "rewards/cosine_scaled_reward": 0.1327524788212031, | |
| "rewards/format_reward": 0.0, | |
| "step": 139 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 511.8605079650879, | |
| "epoch": 1.4258555133079849, | |
| "grad_norm": 0.1867491602897644, | |
| "learning_rate": 2.2967959127220137e-07, | |
| "loss": 0.0068, | |
| "num_tokens": 98728225.0, | |
| "reward": 0.3189204416703433, | |
| "reward_std": 0.3536707144230604, | |
| "rewards/cosine_scaled_reward": 0.3189204225782305, | |
| "rewards/format_reward": 0.0, | |
| "step": 140 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 527.9910926818848, | |
| "epoch": 1.4359949302915083, | |
| "grad_norm": 0.18118509650230408, | |
| "learning_rate": 2.2221488349019902e-07, | |
| "loss": -0.0149, | |
| "num_tokens": 99349665.0, | |
| "reward": 0.136872966773808, | |
| "reward_std": 0.42309295386075974, | |
| "rewards/cosine_scaled_reward": 0.13687296118587255, | |
| "rewards/format_reward": 0.0, | |
| "step": 141 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 508.416316986084, | |
| "epoch": 1.4461343472750317, | |
| "grad_norm": 0.17504200339317322, | |
| "learning_rate": 2.1483868153251788e-07, | |
| "loss": -0.0121, | |
| "num_tokens": 99936750.0, | |
| "reward": 0.24867849051952362, | |
| "reward_std": 0.4239983968436718, | |
| "rewards/cosine_scaled_reward": 0.2486784835346043, | |
| "rewards/format_reward": 0.0, | |
| "step": 142 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 571.9408798217773, | |
| "epoch": 1.456273764258555, | |
| "grad_norm": 0.2036023736000061, | |
| "learning_rate": 2.0755333554967346e-07, | |
| "loss": -0.0038, | |
| "num_tokens": 100586409.0, | |
| "reward": 0.11940343677997589, | |
| "reward_std": 0.5181355699896812, | |
| "rewards/cosine_scaled_reward": 0.11940343119204044, | |
| "rewards/format_reward": 0.0, | |
| "step": 143 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 599.2779235839844, | |
| "epoch": 1.4664131812420786, | |
| "grad_norm": 0.17052923142910004, | |
| "learning_rate": 2.0036116674432652e-07, | |
| "loss": -0.0245, | |
| "num_tokens": 101259810.0, | |
| "reward": 0.12587101384997368, | |
| "reward_std": 0.3964061960577965, | |
| "rewards/cosine_scaled_reward": 0.12587100733071566, | |
| "rewards/format_reward": 0.0, | |
| "step": 144 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 473.37948989868164, | |
| "epoch": 1.476552598225602, | |
| "grad_norm": 0.1972968876361847, | |
| "learning_rate": 1.9326446663172035e-07, | |
| "loss": -0.0245, | |
| "num_tokens": 101815438.0, | |
| "reward": 0.2772935745306313, | |
| "reward_std": 0.44544481486082077, | |
| "rewards/cosine_scaled_reward": 0.2772935628890991, | |
| "rewards/format_reward": 0.0, | |
| "step": 145 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 479.25560760498047, | |
| "epoch": 1.4866920152091254, | |
| "grad_norm": 0.236617311835289, | |
| "learning_rate": 1.8626549630957395e-07, | |
| "loss": 0.0277, | |
| "num_tokens": 102372995.0, | |
| "reward": 0.2525997795164585, | |
| "reward_std": 0.4284665212035179, | |
| "rewards/cosine_scaled_reward": 0.25259976275265217, | |
| "rewards/format_reward": 0.0, | |
| "step": 146 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 550.899585723877, | |
| "epoch": 1.496831432192649, | |
| "grad_norm": 0.17915715277194977, | |
| "learning_rate": 1.7936648573766954e-07, | |
| "loss": -0.0269, | |
| "num_tokens": 103003313.0, | |
| "reward": 0.16996209789067507, | |
| "reward_std": 0.42602597922086716, | |
| "rewards/cosine_scaled_reward": 0.1699620874132961, | |
| "rewards/format_reward": 0.0, | |
| "step": 147 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 564.1919822692871, | |
| "epoch": 1.5069708491761724, | |
| "grad_norm": 0.18789808452129364, | |
| "learning_rate": 1.725696330273575e-07, | |
| "loss": -0.0676, | |
| "num_tokens": 103639173.0, | |
| "reward": 0.16659816866740584, | |
| "reward_std": 0.36924581229686737, | |
| "rewards/cosine_scaled_reward": 0.16659815539605916, | |
| "rewards/format_reward": 0.0, | |
| "step": 148 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 487.6774711608887, | |
| "epoch": 1.5171102661596958, | |
| "grad_norm": 0.20230001211166382, | |
| "learning_rate": 1.65877103741212e-07, | |
| "loss": 0.0106, | |
| "num_tokens": 104200860.0, | |
| "reward": 0.2995794331654906, | |
| "reward_std": 0.4441289193928242, | |
| "rewards/cosine_scaled_reward": 0.2995794131420553, | |
| "rewards/format_reward": 0.0, | |
| "step": 149 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 515.7678756713867, | |
| "epoch": 1.5272496831432192, | |
| "grad_norm": 0.20191656053066254, | |
| "learning_rate": 1.592910302030544e-07, | |
| "loss": 0.0207, | |
| "num_tokens": 104782292.0, | |
| "reward": 0.17506056488491595, | |
| "reward_std": 0.4823239389806986, | |
| "rewards/cosine_scaled_reward": 0.17506056022830307, | |
| "rewards/format_reward": 0.0, | |
| "step": 150 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 517.0513648986816, | |
| "epoch": 1.5373891001267426, | |
| "grad_norm": 0.18813706934452057, | |
| "learning_rate": 1.5281351081856976e-07, | |
| "loss": 0.0014, | |
| "num_tokens": 105376826.0, | |
| "reward": 0.19976339329605253, | |
| "reward_std": 0.43244756758213043, | |
| "rewards/cosine_scaled_reward": 0.19976338025753648, | |
| "rewards/format_reward": 0.0, | |
| "step": 151 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 517.2968978881836, | |
| "epoch": 1.5475285171102662, | |
| "grad_norm": 0.18564638495445251, | |
| "learning_rate": 1.4644660940672627e-07, | |
| "loss": -0.0032, | |
| "num_tokens": 105965916.0, | |
| "reward": 0.29871877538971603, | |
| "reward_std": 0.39689321629703045, | |
| "rewards/cosine_scaled_reward": 0.29871875676326454, | |
| "rewards/format_reward": 0.0, | |
| "step": 152 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 506.71319580078125, | |
| "epoch": 1.5576679340937896, | |
| "grad_norm": 0.17704474925994873, | |
| "learning_rate": 1.4019235454221856e-07, | |
| "loss": -0.0185, | |
| "num_tokens": 106554003.0, | |
| "reward": 0.21053828485310078, | |
| "reward_std": 0.3987581580877304, | |
| "rewards/cosine_scaled_reward": 0.21053827833384275, | |
| "rewards/format_reward": 0.0, | |
| "step": 153 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 475.9431037902832, | |
| "epoch": 1.5678073510773132, | |
| "grad_norm": 0.19738495349884033, | |
| "learning_rate": 1.3405273890913737e-07, | |
| "loss": 0.0262, | |
| "num_tokens": 107102464.0, | |
| "reward": 0.21143945213407278, | |
| "reward_std": 0.42565931007266045, | |
| "rewards/cosine_scaled_reward": 0.21143944095820189, | |
| "rewards/format_reward": 0.0, | |
| "step": 154 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 508.6618537902832, | |
| "epoch": 1.5779467680608366, | |
| "grad_norm": 0.328647643327713, | |
| "learning_rate": 1.280297186660752e-07, | |
| "loss": 0.0234, | |
| "num_tokens": 107676601.0, | |
| "reward": 0.25387828424572945, | |
| "reward_std": 0.4254852868616581, | |
| "rewards/cosine_scaled_reward": 0.25387826934456825, | |
| "rewards/format_reward": 0.0, | |
| "step": 155 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 587.3850631713867, | |
| "epoch": 1.58808618504436, | |
| "grad_norm": 0.17507341504096985, | |
| "learning_rate": 1.2212521282287093e-07, | |
| "loss": -0.0302, | |
| "num_tokens": 108322826.0, | |
| "reward": 0.1246091895736754, | |
| "reward_std": 0.43387965485453606, | |
| "rewards/cosine_scaled_reward": 0.12460918258875608, | |
| "rewards/format_reward": 0.0, | |
| "step": 156 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 570.8884086608887, | |
| "epoch": 1.5982256020278833, | |
| "grad_norm": 0.17769315838813782, | |
| "learning_rate": 1.1634110262918717e-07, | |
| "loss": -0.0209, | |
| "num_tokens": 108978350.0, | |
| "reward": 0.17509674560278654, | |
| "reward_std": 0.3465359937399626, | |
| "rewards/cosine_scaled_reward": 0.17509673908352852, | |
| "rewards/format_reward": 0.0, | |
| "step": 157 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 546.3772621154785, | |
| "epoch": 1.6083650190114067, | |
| "grad_norm": 0.18788230419158936, | |
| "learning_rate": 1.1067923097512255e-07, | |
| "loss": -0.0358, | |
| "num_tokens": 109598616.0, | |
| "reward": 0.12754715513437986, | |
| "reward_std": 0.4496493525803089, | |
| "rewards/cosine_scaled_reward": 0.12754714622860774, | |
| "rewards/format_reward": 0.0, | |
| "step": 158 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 543.7288246154785, | |
| "epoch": 1.6185044359949303, | |
| "grad_norm": 0.2574346661567688, | |
| "learning_rate": 1.0514140180404202e-07, | |
| "loss": -0.0512, | |
| "num_tokens": 110214853.0, | |
| "reward": 0.08189231436699629, | |
| "reward_std": 0.41547736898064613, | |
| "rewards/cosine_scaled_reward": 0.08189231110736728, | |
| "rewards/format_reward": 0.0, | |
| "step": 159 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 607.2890892028809, | |
| "epoch": 1.6286438529784537, | |
| "grad_norm": 0.22311408817768097, | |
| "learning_rate": 9.972937953781984e-08, | |
| "loss": -0.0035, | |
| "num_tokens": 110905952.0, | |
| "reward": 0.07692541873257142, | |
| "reward_std": 0.4941936247050762, | |
| "rewards/cosine_scaled_reward": 0.07692541686992627, | |
| "rewards/format_reward": 0.0, | |
| "step": 160 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 528.2031402587891, | |
| "epoch": 1.6387832699619773, | |
| "grad_norm": 1.5113683938980103, | |
| "learning_rate": 9.444488851467041e-08, | |
| "loss": -0.0166, | |
| "num_tokens": 111496894.0, | |
| "reward": 0.23110088612884283, | |
| "reward_std": 0.44028190523386, | |
| "rewards/cosine_scaled_reward": 0.23110087541863322, | |
| "rewards/format_reward": 0.0, | |
| "step": 161 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 523.2132034301758, | |
| "epoch": 1.6489226869455007, | |
| "grad_norm": 0.18456611037254333, | |
| "learning_rate": 8.928961243975436e-08, | |
| "loss": -0.0148, | |
| "num_tokens": 112098549.0, | |
| "reward": 0.16438766405917704, | |
| "reward_std": 0.4540070556104183, | |
| "rewards/cosine_scaled_reward": 0.16713074478320777, | |
| "rewards/format_reward": 0.0, | |
| "step": 162 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 490.20092010498047, | |
| "epoch": 1.659062103929024, | |
| "grad_norm": 0.20410823822021484, | |
| "learning_rate": 8.426519384872732e-08, | |
| "loss": -0.0209, | |
| "num_tokens": 112659905.0, | |
| "reward": 0.2585650213295594, | |
| "reward_std": 0.42336183600127697, | |
| "rewards/cosine_scaled_reward": 0.25856500840745866, | |
| "rewards/format_reward": 0.0, | |
| "step": 163 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 515.3370704650879, | |
| "epoch": 1.6692015209125475, | |
| "grad_norm": 0.2762053906917572, | |
| "learning_rate": 7.937323358440934e-08, | |
| "loss": 0.036, | |
| "num_tokens": 113252679.0, | |
| "reward": 0.21294681075960398, | |
| "reward_std": 0.4499225113540888, | |
| "rewards/cosine_scaled_reward": 0.2129468061029911, | |
| "rewards/format_reward": 0.0, | |
| "step": 164 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 517.8939971923828, | |
| "epoch": 1.6793409378960709, | |
| "grad_norm": 0.18798772990703583, | |
| "learning_rate": 7.461529028673463e-08, | |
| "loss": 0.0349, | |
| "num_tokens": 113849520.0, | |
| "reward": 0.2390075612347573, | |
| "reward_std": 0.4681055396795273, | |
| "rewards/cosine_scaled_reward": 0.23900754831265658, | |
| "rewards/format_reward": 0.0, | |
| "step": 165 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 557.4988975524902, | |
| "epoch": 1.6894803548795945, | |
| "grad_norm": 0.19520263373851776, | |
| "learning_rate": 6.999287989614971e-08, | |
| "loss": 0.0068, | |
| "num_tokens": 114481479.0, | |
| "reward": 0.11839688033796847, | |
| "reward_std": 0.5270655192434788, | |
| "rewards/cosine_scaled_reward": 0.11839687242172658, | |
| "rewards/format_reward": 0.0, | |
| "step": 166 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 484.58261489868164, | |
| "epoch": 1.6996197718631179, | |
| "grad_norm": 0.21452556550502777, | |
| "learning_rate": 6.550747517061656e-08, | |
| "loss": 0.0029, | |
| "num_tokens": 115042209.0, | |
| "reward": 0.24680990353226662, | |
| "reward_std": 0.4576365016400814, | |
| "rewards/cosine_scaled_reward": 0.2468099007382989, | |
| "rewards/format_reward": 0.0, | |
| "step": 167 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 516.2667655944824, | |
| "epoch": 1.7097591888466415, | |
| "grad_norm": 0.18506427109241486, | |
| "learning_rate": 6.116050521637218e-08, | |
| "loss": 0.019, | |
| "num_tokens": 115633112.0, | |
| "reward": 0.22252054838463664, | |
| "reward_std": 0.4747694581747055, | |
| "rewards/cosine_scaled_reward": 0.22252054512500763, | |
| "rewards/format_reward": 0.0, | |
| "step": 168 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 511.237735748291, | |
| "epoch": 1.7198986058301649, | |
| "grad_norm": 0.19289632141590118, | |
| "learning_rate": 5.6953355032598795e-08, | |
| "loss": -0.0568, | |
| "num_tokens": 116228253.0, | |
| "reward": 0.08733667735941708, | |
| "reward_std": 0.49206770583987236, | |
| "rewards/cosine_scaled_reward": 0.08733667538035661, | |
| "rewards/format_reward": 0.0, | |
| "step": 169 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 520.1964530944824, | |
| "epoch": 1.7300380228136882, | |
| "grad_norm": 0.19966185092926025, | |
| "learning_rate": 5.288736507014435e-08, | |
| "loss": -0.0308, | |
| "num_tokens": 116818349.0, | |
| "reward": 0.20070420764386654, | |
| "reward_std": 0.45568813756108284, | |
| "rewards/cosine_scaled_reward": 0.20070419739931822, | |
| "rewards/format_reward": 0.0, | |
| "step": 170 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 482.51565170288086, | |
| "epoch": 1.7401774397972116, | |
| "grad_norm": 0.2040020376443863, | |
| "learning_rate": 4.896383080443933e-08, | |
| "loss": -0.0247, | |
| "num_tokens": 117375435.0, | |
| "reward": 0.13767189331701957, | |
| "reward_std": 0.42815835028886795, | |
| "rewards/cosine_scaled_reward": 0.13767188772908412, | |
| "rewards/format_reward": 0.0, | |
| "step": 171 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 504.1607437133789, | |
| "epoch": 1.750316856780735, | |
| "grad_norm": 0.20132046937942505, | |
| "learning_rate": 4.518400232274078e-08, | |
| "loss": -0.0345, | |
| "num_tokens": 117957667.0, | |
| "reward": 0.17123566241934896, | |
| "reward_std": 0.3756715804338455, | |
| "rewards/cosine_scaled_reward": 0.17370464373379946, | |
| "rewards/format_reward": 0.0, | |
| "step": 172 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 460.175235748291, | |
| "epoch": 1.7604562737642584, | |
| "grad_norm": 0.2837740480899811, | |
| "learning_rate": 4.1549083925840165e-08, | |
| "loss": -0.0105, | |
| "num_tokens": 118487432.0, | |
| "reward": 0.2758436817675829, | |
| "reward_std": 0.4501863084733486, | |
| "rewards/cosine_scaled_reward": 0.2758436668664217, | |
| "rewards/format_reward": 0.0, | |
| "step": 173 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 496.6093940734863, | |
| "epoch": 1.770595690747782, | |
| "grad_norm": 0.18853703141212463, | |
| "learning_rate": 3.806023374435663e-08, | |
| "loss": 0.0034, | |
| "num_tokens": 119060922.0, | |
| "reward": 0.19881045899819583, | |
| "reward_std": 0.4718875251710415, | |
| "rewards/cosine_scaled_reward": 0.19881044793874025, | |
| "rewards/format_reward": 0.0, | |
| "step": 174 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 499.9899787902832, | |
| "epoch": 1.7807351077313056, | |
| "grad_norm": 0.18940913677215576, | |
| "learning_rate": 3.4718563369743213e-08, | |
| "loss": -0.0418, | |
| "num_tokens": 119642241.0, | |
| "reward": 0.1533535374328494, | |
| "reward_std": 0.45827095210552216, | |
| "rewards/cosine_scaled_reward": 0.1533535262569785, | |
| "rewards/format_reward": 0.0, | |
| "step": 175 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 483.29467010498047, | |
| "epoch": 1.790874524714829, | |
| "grad_norm": 0.3907623291015625, | |
| "learning_rate": 3.15251375001192e-08, | |
| "loss": 0.0006, | |
| "num_tokens": 120215377.0, | |
| "reward": 0.30862145498394966, | |
| "reward_std": 0.44223641976714134, | |
| "rewards/cosine_scaled_reward": 0.308621431235224, | |
| "rewards/format_reward": 0.0, | |
| "step": 176 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 515.3270263671875, | |
| "epoch": 1.8010139416983524, | |
| "grad_norm": 0.18325906991958618, | |
| "learning_rate": 2.8480973601043955e-08, | |
| "loss": -0.0398, | |
| "num_tokens": 120811918.0, | |
| "reward": 0.24171121884137392, | |
| "reward_std": 0.36342719942331314, | |
| "rewards/cosine_scaled_reward": 0.24171120021492243, | |
| "rewards/format_reward": 0.0, | |
| "step": 177 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 542.7254676818848, | |
| "epoch": 1.8111533586818758, | |
| "grad_norm": 0.17345182597637177, | |
| "learning_rate": 2.558704158134023e-08, | |
| "loss": -0.0589, | |
| "num_tokens": 121438224.0, | |
| "reward": 0.10904384450986981, | |
| "reward_std": 0.4528394937515259, | |
| "rewards/cosine_scaled_reward": 0.10904384264722466, | |
| "rewards/format_reward": 0.0, | |
| "step": 178 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 543.6852951049805, | |
| "epoch": 1.8212927756653992, | |
| "grad_norm": 0.1887708306312561, | |
| "learning_rate": 2.2844263484068093e-08, | |
| "loss": -0.0194, | |
| "num_tokens": 122052366.0, | |
| "reward": 0.1703888149932027, | |
| "reward_std": 0.4111927505582571, | |
| "rewards/cosine_scaled_reward": 0.17038880474865437, | |
| "rewards/format_reward": 0.0, | |
| "step": 179 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 485.0948905944824, | |
| "epoch": 1.8314321926489225, | |
| "grad_norm": 0.20806105434894562, | |
| "learning_rate": 2.025351319275137e-08, | |
| "loss": -0.0092, | |
| "num_tokens": 122617979.0, | |
| "reward": 0.22041516425088048, | |
| "reward_std": 0.39602686278522015, | |
| "rewards/cosine_scaled_reward": 0.22041516285389662, | |
| "rewards/format_reward": 0.0, | |
| "step": 180 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 521.0446624755859, | |
| "epoch": 1.8415716096324461, | |
| "grad_norm": 0.1908193826675415, | |
| "learning_rate": 1.781561615294652e-08, | |
| "loss": -0.0301, | |
| "num_tokens": 123219299.0, | |
| "reward": 0.1940143120009452, | |
| "reward_std": 0.4348856508731842, | |
| "rewards/cosine_scaled_reward": 0.19401429709978402, | |
| "rewards/format_reward": 0.0, | |
| "step": 181 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 570.9743461608887, | |
| "epoch": 1.8517110266159695, | |
| "grad_norm": 0.17504295706748962, | |
| "learning_rate": 1.553134910924636e-08, | |
| "loss": -0.0429, | |
| "num_tokens": 123866996.0, | |
| "reward": 0.08305117121199146, | |
| "reward_std": 0.41529295966029167, | |
| "rewards/cosine_scaled_reward": 0.08305116754490882, | |
| "rewards/format_reward": 0.0, | |
| "step": 182 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 481.1294822692871, | |
| "epoch": 1.8618504435994931, | |
| "grad_norm": 0.20204633474349976, | |
| "learning_rate": 1.340143985779829e-08, | |
| "loss": -0.0051, | |
| "num_tokens": 124424848.0, | |
| "reward": 0.2932664602994919, | |
| "reward_std": 0.4205982144922018, | |
| "rewards/cosine_scaled_reward": 0.2932664528489113, | |
| "rewards/format_reward": 0.0, | |
| "step": 183 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 498.0591735839844, | |
| "epoch": 1.8719898605830165, | |
| "grad_norm": 0.18216805160045624, | |
| "learning_rate": 1.1426567014420297e-08, | |
| "loss": -0.0097, | |
| "num_tokens": 124997813.0, | |
| "reward": 0.27011361718177795, | |
| "reward_std": 0.42340338230133057, | |
| "rewards/cosine_scaled_reward": 0.27011359576135874, | |
| "rewards/format_reward": 0.0, | |
| "step": 184 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 513.774585723877, | |
| "epoch": 1.88212927756654, | |
| "grad_norm": 0.1906953901052475, | |
| "learning_rate": 9.607359798384784e-09, | |
| "loss": -0.0212, | |
| "num_tokens": 125573635.0, | |
| "reward": 0.15609056316316128, | |
| "reward_std": 0.46621261164546013, | |
| "rewards/cosine_scaled_reward": 0.15609055384993553, | |
| "rewards/format_reward": 0.0, | |
| "step": 185 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 511.1183280944824, | |
| "epoch": 1.8922686945500633, | |
| "grad_norm": 0.20339244604110718, | |
| "learning_rate": 7.944397831941951e-09, | |
| "loss": 0.0108, | |
| "num_tokens": 126178093.0, | |
| "reward": 0.12280734069645405, | |
| "reward_std": 0.4304603859782219, | |
| "rewards/cosine_scaled_reward": 0.12280733417719603, | |
| "rewards/format_reward": 0.0, | |
| "step": 186 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 583.9364051818848, | |
| "epoch": 1.9024081115335867, | |
| "grad_norm": 0.18367899954319, | |
| "learning_rate": 6.438210955644452e-09, | |
| "loss": -0.0444, | |
| "num_tokens": 126843724.0, | |
| "reward": 0.019926082342863083, | |
| "reward_std": 0.4689461179077625, | |
| "rewards/cosine_scaled_reward": 0.019926077569834888, | |
| "rewards/format_reward": 0.0, | |
| "step": 187 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 523.2076110839844, | |
| "epoch": 1.9125475285171103, | |
| "grad_norm": 0.17626607418060303, | |
| "learning_rate": 5.0892790595336575e-09, | |
| "loss": -0.0197, | |
| "num_tokens": 127449534.0, | |
| "reward": 0.11797729413956404, | |
| "reward_std": 0.497987262904644, | |
| "rewards/cosine_scaled_reward": 0.11797728785313666, | |
| "rewards/format_reward": 0.0, | |
| "step": 188 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 525.0011367797852, | |
| "epoch": 1.9226869455006337, | |
| "grad_norm": 0.1973433792591095, | |
| "learning_rate": 3.898031930240797e-09, | |
| "loss": -0.029, | |
| "num_tokens": 128044935.0, | |
| "reward": 0.10833488963544369, | |
| "reward_std": 0.4491695165634155, | |
| "rewards/cosine_scaled_reward": 0.1083348891697824, | |
| "rewards/format_reward": 0.0, | |
| "step": 189 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 548.4966850280762, | |
| "epoch": 1.9328263624841573, | |
| "grad_norm": 0.17003753781318665, | |
| "learning_rate": 2.8648491140513264e-09, | |
| "loss": 0.0062, | |
| "num_tokens": 128662836.0, | |
| "reward": 0.20282268431037664, | |
| "reward_std": 0.4673178493976593, | |
| "rewards/cosine_scaled_reward": 0.20282267965376377, | |
| "rewards/format_reward": 0.0, | |
| "step": 190 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 440.6149711608887, | |
| "epoch": 1.9429657794676807, | |
| "grad_norm": 0.21727675199508667, | |
| "learning_rate": 1.9900597959770505e-09, | |
| "loss": 0.0655, | |
| "num_tokens": 129182755.0, | |
| "reward": 0.40362408570945263, | |
| "reward_std": 0.34423849545419216, | |
| "rewards/cosine_scaled_reward": 0.403624065220356, | |
| "rewards/format_reward": 0.0, | |
| "step": 191 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 541.2031555175781, | |
| "epoch": 1.953105196451204, | |
| "grad_norm": 0.19199801981449127, | |
| "learning_rate": 1.2739426948732424e-09, | |
| "loss": -0.0442, | |
| "num_tokens": 129801641.0, | |
| "reward": 0.19715721579268575, | |
| "reward_std": 0.4420705772936344, | |
| "rewards/cosine_scaled_reward": 0.19715720368549228, | |
| "rewards/format_reward": 0.0, | |
| "step": 192 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 526.9609565734863, | |
| "epoch": 1.9632446134347274, | |
| "grad_norm": 0.22452791035175323, | |
| "learning_rate": 7.16725974635568e-10, | |
| "loss": 0.0124, | |
| "num_tokens": 130399638.0, | |
| "reward": 0.20976402005180717, | |
| "reward_std": 0.4755300283432007, | |
| "rewards/cosine_scaled_reward": 0.20976400654762983, | |
| "rewards/format_reward": 0.0, | |
| "step": 193 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 550.6752510070801, | |
| "epoch": 1.9733840304182508, | |
| "grad_norm": 0.18797193467617035, | |
| "learning_rate": 3.185871715041255e-10, | |
| "loss": -0.035, | |
| "num_tokens": 131025891.0, | |
| "reward": 0.1596639035269618, | |
| "reward_std": 0.4088666681200266, | |
| "rewards/cosine_scaled_reward": 0.15966389642562717, | |
| "rewards/format_reward": 0.0, | |
| "step": 194 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 447.8102798461914, | |
| "epoch": 1.9835234474017744, | |
| "grad_norm": 0.19335201382637024, | |
| "learning_rate": 7.96531374983589e-11, | |
| "loss": -0.0162, | |
| "num_tokens": 131547609.0, | |
| "reward": 0.2565318220295012, | |
| "reward_std": 0.46955981478095055, | |
| "rewards/cosine_scaled_reward": 0.25653181644156575, | |
| "rewards/format_reward": 0.0, | |
| "step": 195 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 516.494140625, | |
| "epoch": 1.9936628643852978, | |
| "grad_norm": 0.180581733584404, | |
| "learning_rate": 0.0, | |
| "loss": -0.0093, | |
| "num_tokens": 132131440.0, | |
| "reward": 0.08059131819754839, | |
| "reward_std": 0.5266188904643059, | |
| "rewards/cosine_scaled_reward": 0.08059131726622581, | |
| "rewards/format_reward": 0.0, | |
| "step": 196 | |
| }, | |
| { | |
| "epoch": 1.9936628643852978, | |
| "step": 196, | |
| "total_flos": 0.0, | |
| "train_loss": -0.019965038798853137, | |
| "train_runtime": 31425.8764, | |
| "train_samples_per_second": 0.703, | |
| "train_steps_per_second": 0.006 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 196, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 2, | |
| "save_steps": 500, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 16, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |