| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.9880609304240429, | |
| "eval_steps": 500, | |
| "global_step": 75, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.013174145738987238, | |
| "grad_norm": 1.194186806678772, | |
| "learning_rate": 6.25e-08, | |
| "logits/chosen": 9.990612030029297, | |
| "logits/rejected": 10.698101997375488, | |
| "logps/chosen": -102.88545989990234, | |
| "logps/ref_chosen": -102.88545989990234, | |
| "logps/ref_rejected": -121.84871673583984, | |
| "logps/rejected": -121.84871673583984, | |
| "loss": 0.3675, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "sft_loss": 0.36753880977630615, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.026348291477974475, | |
| "grad_norm": 0.5353251099586487, | |
| "learning_rate": 1.25e-07, | |
| "logits/chosen": 10.211905479431152, | |
| "logits/rejected": 11.06594467163086, | |
| "logps/chosen": -107.70349884033203, | |
| "logps/ref_chosen": -107.70349884033203, | |
| "logps/ref_rejected": -121.89966583251953, | |
| "logps/rejected": -121.89966583251953, | |
| "loss": 0.4101, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "sft_loss": 0.41013145446777344, | |
| "step": 2 | |
| }, | |
| { | |
| "epoch": 0.03952243721696171, | |
| "grad_norm": 0.7126303315162659, | |
| "learning_rate": 1.875e-07, | |
| "logits/chosen": 10.032384872436523, | |
| "logits/rejected": 11.023520469665527, | |
| "logps/chosen": -108.3123779296875, | |
| "logps/ref_chosen": -107.98188781738281, | |
| "logps/ref_rejected": -124.51527404785156, | |
| "logps/rejected": -124.87130737304688, | |
| "loss": 0.412, | |
| "rewards/accuracies": 0.5234375, | |
| "rewards/chosen": -0.003304910147562623, | |
| "rewards/margins": 0.0002554532838985324, | |
| "rewards/rejected": -0.003560363780707121, | |
| "sft_loss": 0.41195932030677795, | |
| "step": 3 | |
| }, | |
| { | |
| "epoch": 0.05269658295594895, | |
| "grad_norm": 1.2344533205032349, | |
| "learning_rate": 2.5e-07, | |
| "logits/chosen": 9.836658477783203, | |
| "logits/rejected": 10.855621337890625, | |
| "logps/chosen": -109.55919647216797, | |
| "logps/ref_chosen": -109.20836639404297, | |
| "logps/ref_rejected": -119.23908996582031, | |
| "logps/rejected": -119.48279571533203, | |
| "loss": 0.4039, | |
| "rewards/accuracies": 0.4921875, | |
| "rewards/chosen": -0.003508324269205332, | |
| "rewards/margins": -0.0010712125804275274, | |
| "rewards/rejected": -0.0024371116887778044, | |
| "sft_loss": 0.4038863480091095, | |
| "step": 4 | |
| }, | |
| { | |
| "epoch": 0.06587072869493618, | |
| "grad_norm": 1.426048994064331, | |
| "learning_rate": 3.1249999999999997e-07, | |
| "logits/chosen": 10.212320327758789, | |
| "logits/rejected": 10.966379165649414, | |
| "logps/chosen": -103.76991271972656, | |
| "logps/ref_chosen": -103.87680053710938, | |
| "logps/ref_rejected": -118.41618347167969, | |
| "logps/rejected": -118.23270416259766, | |
| "loss": 0.3697, | |
| "rewards/accuracies": 0.453125, | |
| "rewards/chosen": 0.0010687037138268352, | |
| "rewards/margins": -0.000766113749705255, | |
| "rewards/rejected": 0.0018348174635320902, | |
| "sft_loss": 0.3697226345539093, | |
| "step": 5 | |
| }, | |
| { | |
| "epoch": 0.07904487443392343, | |
| "grad_norm": 1.413549780845642, | |
| "learning_rate": 3.75e-07, | |
| "logits/chosen": 10.700042724609375, | |
| "logits/rejected": 11.478326797485352, | |
| "logps/chosen": -107.56877899169922, | |
| "logps/ref_chosen": -107.58968353271484, | |
| "logps/ref_rejected": -122.07303619384766, | |
| "logps/rejected": -121.85940551757812, | |
| "loss": 0.3909, | |
| "rewards/accuracies": 0.4609375, | |
| "rewards/chosen": 0.0002090137859340757, | |
| "rewards/margins": -0.0019273017533123493, | |
| "rewards/rejected": 0.00213631521910429, | |
| "sft_loss": 0.390906423330307, | |
| "step": 6 | |
| }, | |
| { | |
| "epoch": 0.09221902017291066, | |
| "grad_norm": 1.2342580556869507, | |
| "learning_rate": 4.375e-07, | |
| "logits/chosen": 10.01632308959961, | |
| "logits/rejected": 10.7178955078125, | |
| "logps/chosen": -107.01339721679688, | |
| "logps/ref_chosen": -107.42727661132812, | |
| "logps/ref_rejected": -116.87063598632812, | |
| "logps/rejected": -116.37357330322266, | |
| "loss": 0.3747, | |
| "rewards/accuracies": 0.4453125, | |
| "rewards/chosen": 0.00413867924362421, | |
| "rewards/margins": -0.0008318667532876134, | |
| "rewards/rejected": 0.004970546346157789, | |
| "sft_loss": 0.3746669888496399, | |
| "step": 7 | |
| }, | |
| { | |
| "epoch": 0.1053931659118979, | |
| "grad_norm": 0.6644937992095947, | |
| "learning_rate": 5e-07, | |
| "logits/chosen": 10.211028099060059, | |
| "logits/rejected": 11.11027717590332, | |
| "logps/chosen": -104.41184997558594, | |
| "logps/ref_chosen": -105.60282135009766, | |
| "logps/ref_rejected": -119.53916931152344, | |
| "logps/rejected": -118.27430725097656, | |
| "loss": 0.3773, | |
| "rewards/accuracies": 0.4921875, | |
| "rewards/chosen": 0.01190974935889244, | |
| "rewards/margins": -0.0007388982339762151, | |
| "rewards/rejected": 0.012648648582398891, | |
| "sft_loss": 0.37729793787002563, | |
| "step": 8 | |
| }, | |
| { | |
| "epoch": 0.11856731165088513, | |
| "grad_norm": 0.9437576532363892, | |
| "learning_rate": 4.997252228714278e-07, | |
| "logits/chosen": 10.179821014404297, | |
| "logits/rejected": 11.147579193115234, | |
| "logps/chosen": -104.13174438476562, | |
| "logps/ref_chosen": -105.46086120605469, | |
| "logps/ref_rejected": -119.00373840332031, | |
| "logps/rejected": -117.734130859375, | |
| "loss": 0.3807, | |
| "rewards/accuracies": 0.5234375, | |
| "rewards/chosen": 0.013291322626173496, | |
| "rewards/margins": 0.0005952615174464881, | |
| "rewards/rejected": 0.012696062214672565, | |
| "sft_loss": 0.38070446252822876, | |
| "step": 9 | |
| }, | |
| { | |
| "epoch": 0.13174145738987236, | |
| "grad_norm": 0.700039803981781, | |
| "learning_rate": 4.989014955054745e-07, | |
| "logits/chosen": 10.076737403869629, | |
| "logits/rejected": 10.897785186767578, | |
| "logps/chosen": -100.81087493896484, | |
| "logps/ref_chosen": -104.21009826660156, | |
| "logps/ref_rejected": -118.9209213256836, | |
| "logps/rejected": -115.75495910644531, | |
| "loss": 0.3367, | |
| "rewards/accuracies": 0.5546875, | |
| "rewards/chosen": 0.033992186188697815, | |
| "rewards/margins": 0.0023326175287365913, | |
| "rewards/rejected": 0.0316595658659935, | |
| "sft_loss": 0.33672136068344116, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.14491560312885962, | |
| "grad_norm": 0.9160856008529663, | |
| "learning_rate": 4.975306286336627e-07, | |
| "logits/chosen": 9.973880767822266, | |
| "logits/rejected": 11.158487319946289, | |
| "logps/chosen": -101.3505630493164, | |
| "logps/ref_chosen": -105.94319152832031, | |
| "logps/ref_rejected": -122.76007843017578, | |
| "logps/rejected": -118.6338119506836, | |
| "loss": 0.3851, | |
| "rewards/accuracies": 0.578125, | |
| "rewards/chosen": 0.045926500111818314, | |
| "rewards/margins": 0.004663803614675999, | |
| "rewards/rejected": 0.04126270115375519, | |
| "sft_loss": 0.3850533962249756, | |
| "step": 11 | |
| }, | |
| { | |
| "epoch": 0.15808974886784685, | |
| "grad_norm": 0.9421964883804321, | |
| "learning_rate": 4.956156357188939e-07, | |
| "logits/chosen": 9.908226013183594, | |
| "logits/rejected": 10.598045349121094, | |
| "logps/chosen": -103.32762908935547, | |
| "logps/ref_chosen": -109.08442687988281, | |
| "logps/ref_rejected": -121.41947174072266, | |
| "logps/rejected": -115.84996795654297, | |
| "loss": 0.3532, | |
| "rewards/accuracies": 0.4921875, | |
| "rewards/chosen": 0.05756799131631851, | |
| "rewards/margins": 0.0018730255542322993, | |
| "rewards/rejected": 0.05569496005773544, | |
| "sft_loss": 0.3532242476940155, | |
| "step": 12 | |
| }, | |
| { | |
| "epoch": 0.17126389460683408, | |
| "grad_norm": 0.3328304886817932, | |
| "learning_rate": 4.931607263312032e-07, | |
| "logits/chosen": 9.964012145996094, | |
| "logits/rejected": 11.03992748260498, | |
| "logps/chosen": -98.97601318359375, | |
| "logps/ref_chosen": -104.62150573730469, | |
| "logps/ref_rejected": -119.55384063720703, | |
| "logps/rejected": -114.12371826171875, | |
| "loss": 0.3686, | |
| "rewards/accuracies": 0.4921875, | |
| "rewards/chosen": 0.056454867124557495, | |
| "rewards/margins": 0.0021536569111049175, | |
| "rewards/rejected": 0.05430121719837189, | |
| "sft_loss": 0.36859023571014404, | |
| "step": 13 | |
| }, | |
| { | |
| "epoch": 0.1844380403458213, | |
| "grad_norm": 0.30642038583755493, | |
| "learning_rate": 4.9017129689421e-07, | |
| "logits/chosen": 10.519927978515625, | |
| "logits/rejected": 11.649580001831055, | |
| "logps/chosen": -96.5634765625, | |
| "logps/ref_chosen": -106.179443359375, | |
| "logps/ref_rejected": -120.73036193847656, | |
| "logps/rejected": -110.86133575439453, | |
| "loss": 0.3385, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.09615952521562576, | |
| "rewards/margins": -0.002530643017962575, | |
| "rewards/rejected": 0.09869016706943512, | |
| "sft_loss": 0.338548868894577, | |
| "step": 14 | |
| }, | |
| { | |
| "epoch": 0.19761218608480857, | |
| "grad_norm": 0.301073282957077, | |
| "learning_rate": 4.866539188226085e-07, | |
| "logits/chosen": 9.891039848327637, | |
| "logits/rejected": 10.824172973632812, | |
| "logps/chosen": -95.14861297607422, | |
| "logps/ref_chosen": -105.70547485351562, | |
| "logps/ref_rejected": -118.89997863769531, | |
| "logps/rejected": -108.2326889038086, | |
| "loss": 0.3305, | |
| "rewards/accuracies": 0.5078125, | |
| "rewards/chosen": 0.10556865483522415, | |
| "rewards/margins": -0.0011043368140235543, | |
| "rewards/rejected": 0.1066729873418808, | |
| "sft_loss": 0.3305360674858093, | |
| "step": 15 | |
| }, | |
| { | |
| "epoch": 0.2107863318237958, | |
| "grad_norm": 0.36065343022346497, | |
| "learning_rate": 4.826163240767716e-07, | |
| "logits/chosen": 10.682470321655273, | |
| "logits/rejected": 11.299846649169922, | |
| "logps/chosen": -96.53520202636719, | |
| "logps/ref_chosen": -108.86376953125, | |
| "logps/ref_rejected": -122.1635513305664, | |
| "logps/rejected": -110.50537872314453, | |
| "loss": 0.3484, | |
| "rewards/accuracies": 0.578125, | |
| "rewards/chosen": 0.12328556925058365, | |
| "rewards/margins": 0.006703883409500122, | |
| "rewards/rejected": 0.11658168584108353, | |
| "sft_loss": 0.3484281003475189, | |
| "step": 16 | |
| }, | |
| { | |
| "epoch": 0.22396047756278303, | |
| "grad_norm": 0.5325565934181213, | |
| "learning_rate": 4.780673881662242e-07, | |
| "logits/chosen": 10.187503814697266, | |
| "logits/rejected": 10.843408584594727, | |
| "logps/chosen": -90.1707992553711, | |
| "logps/ref_chosen": -102.93986511230469, | |
| "logps/ref_rejected": -119.43718719482422, | |
| "logps/rejected": -106.76301574707031, | |
| "loss": 0.359, | |
| "rewards/accuracies": 0.4453125, | |
| "rewards/chosen": 0.12769076228141785, | |
| "rewards/margins": 0.0009490540251135826, | |
| "rewards/rejected": 0.1267417073249817, | |
| "sft_loss": 0.3589847683906555, | |
| "step": 17 | |
| }, | |
| { | |
| "epoch": 0.23713462330177026, | |
| "grad_norm": 0.4098409116268158, | |
| "learning_rate": 4.730171106393466e-07, | |
| "logits/chosen": 10.4215669631958, | |
| "logits/rejected": 11.216498374938965, | |
| "logps/chosen": -90.09894561767578, | |
| "logps/ref_chosen": -103.81341552734375, | |
| "logps/ref_rejected": -117.45123291015625, | |
| "logps/rejected": -104.58552551269531, | |
| "loss": 0.3368, | |
| "rewards/accuracies": 0.515625, | |
| "rewards/chosen": 0.13714462518692017, | |
| "rewards/margins": 0.00848748255521059, | |
| "rewards/rejected": 0.12865713238716125, | |
| "sft_loss": 0.33678534626960754, | |
| "step": 18 | |
| }, | |
| { | |
| "epoch": 0.2503087690407575, | |
| "grad_norm": 0.3302735984325409, | |
| "learning_rate": 4.6747659310219757e-07, | |
| "logits/chosen": 10.332744598388672, | |
| "logits/rejected": 11.005766868591309, | |
| "logps/chosen": -94.52428436279297, | |
| "logps/ref_chosen": -107.85797119140625, | |
| "logps/ref_rejected": -121.88042449951172, | |
| "logps/rejected": -108.09265899658203, | |
| "loss": 0.3222, | |
| "rewards/accuracies": 0.4296875, | |
| "rewards/chosen": 0.1333368420600891, | |
| "rewards/margins": -0.004540742840617895, | |
| "rewards/rejected": 0.13787758350372314, | |
| "sft_loss": 0.3221552073955536, | |
| "step": 19 | |
| }, | |
| { | |
| "epoch": 0.2634829147797447, | |
| "grad_norm": 0.40531161427497864, | |
| "learning_rate": 4.6145801481477433e-07, | |
| "logits/chosen": 10.747330665588379, | |
| "logits/rejected": 11.561124801635742, | |
| "logps/chosen": -89.97228240966797, | |
| "logps/ref_chosen": -103.42721557617188, | |
| "logps/ref_rejected": -116.7796630859375, | |
| "logps/rejected": -103.99850463867188, | |
| "loss": 0.3157, | |
| "rewards/accuracies": 0.5703125, | |
| "rewards/chosen": 0.13454943895339966, | |
| "rewards/margins": 0.006737923249602318, | |
| "rewards/rejected": 0.1278115212917328, | |
| "sft_loss": 0.3156886100769043, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.276657060518732, | |
| "grad_norm": 0.2686282992362976, | |
| "learning_rate": 4.549746059183561e-07, | |
| "logits/chosen": 9.720458984375, | |
| "logits/rejected": 10.846506118774414, | |
| "logps/chosen": -92.48249816894531, | |
| "logps/ref_chosen": -106.60163879394531, | |
| "logps/ref_rejected": -124.56562805175781, | |
| "logps/rejected": -109.58876037597656, | |
| "loss": 0.3106, | |
| "rewards/accuracies": 0.46875, | |
| "rewards/chosen": 0.14119136333465576, | |
| "rewards/margins": -0.00857722107321024, | |
| "rewards/rejected": 0.14976857602596283, | |
| "sft_loss": 0.31064528226852417, | |
| "step": 21 | |
| }, | |
| { | |
| "epoch": 0.28983120625771924, | |
| "grad_norm": 0.5989738702774048, | |
| "learning_rate": 4.480406183527823e-07, | |
| "logits/chosen": 10.225810050964355, | |
| "logits/rejected": 11.099544525146484, | |
| "logps/chosen": -88.04141998291016, | |
| "logps/ref_chosen": -103.77696228027344, | |
| "logps/ref_rejected": -118.73616027832031, | |
| "logps/rejected": -104.40451049804688, | |
| "loss": 0.3321, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": 0.157355397939682, | |
| "rewards/margins": 0.014038847759366035, | |
| "rewards/rejected": 0.14331655204296112, | |
| "sft_loss": 0.3321138620376587, | |
| "step": 22 | |
| }, | |
| { | |
| "epoch": 0.3030053519967065, | |
| "grad_norm": 0.1970965415239334, | |
| "learning_rate": 4.4067129452759546e-07, | |
| "logits/chosen": 10.115339279174805, | |
| "logits/rejected": 11.140266418457031, | |
| "logps/chosen": -87.26233673095703, | |
| "logps/ref_chosen": -104.72956085205078, | |
| "logps/ref_rejected": -121.35556030273438, | |
| "logps/rejected": -104.43501281738281, | |
| "loss": 0.3228, | |
| "rewards/accuracies": 0.4921875, | |
| "rewards/chosen": 0.17467224597930908, | |
| "rewards/margins": 0.005466699134558439, | |
| "rewards/rejected": 0.16920553147792816, | |
| "sft_loss": 0.3228015899658203, | |
| "step": 23 | |
| }, | |
| { | |
| "epoch": 0.3161794977356937, | |
| "grad_norm": 0.2894323468208313, | |
| "learning_rate": 4.3288283381591725e-07, | |
| "logits/chosen": 10.147160530090332, | |
| "logits/rejected": 10.98647689819336, | |
| "logps/chosen": -86.99087524414062, | |
| "logps/ref_chosen": -105.88758087158203, | |
| "logps/ref_rejected": -125.69054412841797, | |
| "logps/rejected": -106.15878295898438, | |
| "loss": 0.3069, | |
| "rewards/accuracies": 0.4765625, | |
| "rewards/chosen": 0.1889670193195343, | |
| "rewards/margins": -0.0063507393933832645, | |
| "rewards/rejected": 0.19531774520874023, | |
| "sft_loss": 0.30694928765296936, | |
| "step": 24 | |
| }, | |
| { | |
| "epoch": 0.32935364347468093, | |
| "grad_norm": 0.31463876366615295, | |
| "learning_rate": 4.246923569447104e-07, | |
| "logits/chosen": 10.327369689941406, | |
| "logits/rejected": 11.063910484313965, | |
| "logps/chosen": -87.99880981445312, | |
| "logps/ref_chosen": -110.0761489868164, | |
| "logps/ref_rejected": -129.10540771484375, | |
| "logps/rejected": -107.19017028808594, | |
| "loss": 0.2993, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": 0.22077329456806183, | |
| "rewards/margins": 0.001621072180569172, | |
| "rewards/rejected": 0.21915221214294434, | |
| "sft_loss": 0.2992705702781677, | |
| "step": 25 | |
| }, | |
| { | |
| "epoch": 0.34252778921366817, | |
| "grad_norm": 0.25855836272239685, | |
| "learning_rate": 4.161178683597054e-07, | |
| "logits/chosen": 10.388958930969238, | |
| "logits/rejected": 11.489179611206055, | |
| "logps/chosen": -81.3349609375, | |
| "logps/ref_chosen": -103.74571990966797, | |
| "logps/ref_rejected": -120.73832702636719, | |
| "logps/rejected": -98.57904052734375, | |
| "loss": 0.2909, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.22410757839679718, | |
| "rewards/margins": 0.0025147469714283943, | |
| "rewards/rejected": 0.22159285843372345, | |
| "sft_loss": 0.2909452021121979, | |
| "step": 26 | |
| }, | |
| { | |
| "epoch": 0.3557019349526554, | |
| "grad_norm": 0.5696946382522583, | |
| "learning_rate": 4.0717821664772124e-07, | |
| "logits/chosen": 10.086296081542969, | |
| "logits/rejected": 11.336379051208496, | |
| "logps/chosen": -81.64080810546875, | |
| "logps/ref_chosen": -105.47428131103516, | |
| "logps/ref_rejected": -120.5193099975586, | |
| "logps/rejected": -97.64772033691406, | |
| "loss": 0.3163, | |
| "rewards/accuracies": 0.5390625, | |
| "rewards/chosen": 0.23833464086055756, | |
| "rewards/margins": 0.00961877591907978, | |
| "rewards/rejected": 0.22871585190296173, | |
| "sft_loss": 0.316275417804718, | |
| "step": 27 | |
| }, | |
| { | |
| "epoch": 0.3688760806916426, | |
| "grad_norm": 0.14858409762382507, | |
| "learning_rate": 3.978930531033806e-07, | |
| "logits/chosen": 9.710855484008789, | |
| "logits/rejected": 10.872222900390625, | |
| "logps/chosen": -80.8717269897461, | |
| "logps/ref_chosen": -103.72540283203125, | |
| "logps/ref_rejected": -119.79557800292969, | |
| "logps/rejected": -96.69274139404297, | |
| "loss": 0.2766, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.22853665053844452, | |
| "rewards/margins": -0.0024917693808674812, | |
| "rewards/rejected": 0.23102842271327972, | |
| "sft_loss": 0.2765931785106659, | |
| "step": 28 | |
| }, | |
| { | |
| "epoch": 0.3820502264306299, | |
| "grad_norm": 0.2741422951221466, | |
| "learning_rate": 3.882827885312998e-07, | |
| "logits/chosen": 10.16092586517334, | |
| "logits/rejected": 11.23297119140625, | |
| "logps/chosen": -85.11812591552734, | |
| "logps/ref_chosen": -108.65434265136719, | |
| "logps/ref_rejected": -121.46784973144531, | |
| "logps/rejected": -98.77241516113281, | |
| "loss": 0.2799, | |
| "rewards/accuracies": 0.5078125, | |
| "rewards/chosen": 0.23536208271980286, | |
| "rewards/margins": 0.008407761342823505, | |
| "rewards/rejected": 0.22695434093475342, | |
| "sft_loss": 0.27991783618927, | |
| "step": 29 | |
| }, | |
| { | |
| "epoch": 0.39522437216961714, | |
| "grad_norm": 0.16848187148571014, | |
| "learning_rate": 3.7836854837871044e-07, | |
| "logits/chosen": 10.2907133102417, | |
| "logits/rejected": 11.690597534179688, | |
| "logps/chosen": -78.23504638671875, | |
| "logps/ref_chosen": -103.62174224853516, | |
| "logps/ref_rejected": -126.73807525634766, | |
| "logps/rejected": -102.43669128417969, | |
| "loss": 0.2962, | |
| "rewards/accuracies": 0.546875, | |
| "rewards/chosen": 0.2538670301437378, | |
| "rewards/margins": 0.010853251442313194, | |
| "rewards/rejected": 0.24301378428936005, | |
| "sft_loss": 0.2962155342102051, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.4083985179086044, | |
| "grad_norm": 0.1987890601158142, | |
| "learning_rate": 3.681721262971413e-07, | |
| "logits/chosen": 9.929094314575195, | |
| "logits/rejected": 10.946361541748047, | |
| "logps/chosen": -80.73751831054688, | |
| "logps/ref_chosen": -106.10479736328125, | |
| "logps/ref_rejected": -120.6382827758789, | |
| "logps/rejected": -96.38467407226562, | |
| "loss": 0.2982, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.2536728084087372, | |
| "rewards/margins": 0.011136716231703758, | |
| "rewards/rejected": 0.24253609776496887, | |
| "sft_loss": 0.2981662452220917, | |
| "step": 31 | |
| }, | |
| { | |
| "epoch": 0.4215726636475916, | |
| "grad_norm": 0.1829695999622345, | |
| "learning_rate": 3.577159362352426e-07, | |
| "logits/chosen": 10.097947120666504, | |
| "logits/rejected": 11.477932929992676, | |
| "logps/chosen": -82.30887603759766, | |
| "logps/ref_chosen": -105.99569702148438, | |
| "logps/ref_rejected": -128.34303283691406, | |
| "logps/rejected": -104.14814758300781, | |
| "loss": 0.2848, | |
| "rewards/accuracies": 0.5078125, | |
| "rewards/chosen": 0.2368682324886322, | |
| "rewards/margins": -0.0050805676728487015, | |
| "rewards/rejected": 0.24194881319999695, | |
| "sft_loss": 0.28480714559555054, | |
| "step": 32 | |
| }, | |
| { | |
| "epoch": 0.43474680938657884, | |
| "grad_norm": 0.22964715957641602, | |
| "learning_rate": 3.470229631680624e-07, | |
| "logits/chosen": 10.105993270874023, | |
| "logits/rejected": 10.923880577087402, | |
| "logps/chosen": -81.49787902832031, | |
| "logps/ref_chosen": -105.72196197509766, | |
| "logps/ref_rejected": -121.59507751464844, | |
| "logps/rejected": -97.2413101196289, | |
| "loss": 0.2686, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.242240771651268, | |
| "rewards/margins": -0.0012968010269105434, | |
| "rewards/rejected": 0.24353757500648499, | |
| "sft_loss": 0.2686034142971039, | |
| "step": 33 | |
| }, | |
| { | |
| "epoch": 0.44792095512556607, | |
| "grad_norm": 0.2910502254962921, | |
| "learning_rate": 3.361167125710832e-07, | |
| "logits/chosen": 10.264101028442383, | |
| "logits/rejected": 11.107752799987793, | |
| "logps/chosen": -85.68121337890625, | |
| "logps/ref_chosen": -111.4834976196289, | |
| "logps/ref_rejected": -130.48089599609375, | |
| "logps/rejected": -104.08442687988281, | |
| "loss": 0.3039, | |
| "rewards/accuracies": 0.4609375, | |
| "rewards/chosen": 0.2580227851867676, | |
| "rewards/margins": -0.005941788665950298, | |
| "rewards/rejected": 0.26396459341049194, | |
| "sft_loss": 0.3038797080516815, | |
| "step": 34 | |
| }, | |
| { | |
| "epoch": 0.4610951008645533, | |
| "grad_norm": 0.2656664550304413, | |
| "learning_rate": 3.2502115875008516e-07, | |
| "logits/chosen": 10.529006958007812, | |
| "logits/rejected": 11.506540298461914, | |
| "logps/chosen": -82.62615966796875, | |
| "logps/ref_chosen": -108.9183349609375, | |
| "logps/ref_rejected": -121.32493591308594, | |
| "logps/rejected": -95.86014556884766, | |
| "loss": 0.2751, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": 0.26292186975479126, | |
| "rewards/margins": 0.008274020627140999, | |
| "rewards/rejected": 0.2546478509902954, | |
| "sft_loss": 0.2751036286354065, | |
| "step": 35 | |
| }, | |
| { | |
| "epoch": 0.47426924660354053, | |
| "grad_norm": 0.4242345690727234, | |
| "learning_rate": 3.137606921404191e-07, | |
| "logits/chosen": 10.204312324523926, | |
| "logits/rejected": 10.856239318847656, | |
| "logps/chosen": -81.047607421875, | |
| "logps/ref_chosen": -107.1411361694336, | |
| "logps/ref_rejected": -118.66165161132812, | |
| "logps/rejected": -92.72647094726562, | |
| "loss": 0.2871, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.26093533635139465, | |
| "rewards/margins": 0.0015834786463528872, | |
| "rewards/rejected": 0.25935184955596924, | |
| "sft_loss": 0.28712767362594604, | |
| "step": 36 | |
| }, | |
| { | |
| "epoch": 0.4874433923425278, | |
| "grad_norm": 0.30854037404060364, | |
| "learning_rate": 3.0236006569153616e-07, | |
| "logits/chosen": 10.416954040527344, | |
| "logits/rejected": 11.237515449523926, | |
| "logps/chosen": -80.80354309082031, | |
| "logps/ref_chosen": -106.6348876953125, | |
| "logps/ref_rejected": -121.37834167480469, | |
| "logps/rejected": -94.95623779296875, | |
| "loss": 0.2868, | |
| "rewards/accuracies": 0.5390625, | |
| "rewards/chosen": 0.2583135664463043, | |
| "rewards/margins": -0.0059075187891721725, | |
| "rewards/rejected": 0.26422107219696045, | |
| "sft_loss": 0.2867960035800934, | |
| "step": 37 | |
| }, | |
| { | |
| "epoch": 0.500617538081515, | |
| "grad_norm": 0.38461658358573914, | |
| "learning_rate": 2.9084434045463254e-07, | |
| "logits/chosen": 9.907350540161133, | |
| "logits/rejected": 10.992050170898438, | |
| "logps/chosen": -76.08082580566406, | |
| "logps/ref_chosen": -104.01033782958984, | |
| "logps/ref_rejected": -119.02666473388672, | |
| "logps/rejected": -90.15015411376953, | |
| "loss": 0.2865, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.2792952358722687, | |
| "rewards/margins": -0.009469768032431602, | |
| "rewards/rejected": 0.28876498341560364, | |
| "sft_loss": 0.28654617071151733, | |
| "step": 38 | |
| }, | |
| { | |
| "epoch": 0.5137916838205022, | |
| "grad_norm": 0.32149407267570496, | |
| "learning_rate": 2.7923883049302066e-07, | |
| "logits/chosen": 10.409524917602539, | |
| "logits/rejected": 11.193167686462402, | |
| "logps/chosen": -82.37915802001953, | |
| "logps/ref_chosen": -109.76485443115234, | |
| "logps/ref_rejected": -122.25163269042969, | |
| "logps/rejected": -96.5480728149414, | |
| "loss": 0.2919, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": 0.2738569676876068, | |
| "rewards/margins": 0.016821369528770447, | |
| "rewards/rejected": 0.25703561305999756, | |
| "sft_loss": 0.2919383943080902, | |
| "step": 39 | |
| }, | |
| { | |
| "epoch": 0.5269658295594895, | |
| "grad_norm": 0.48350030183792114, | |
| "learning_rate": 2.6756904723632324e-07, | |
| "logits/chosen": 10.301675796508789, | |
| "logits/rejected": 11.467926025390625, | |
| "logps/chosen": -79.19075012207031, | |
| "logps/ref_chosen": -107.18782806396484, | |
| "logps/ref_rejected": -124.24542236328125, | |
| "logps/rejected": -96.9850082397461, | |
| "loss": 0.2805, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.2799707055091858, | |
| "rewards/margins": 0.007366571109741926, | |
| "rewards/rejected": 0.27260416746139526, | |
| "sft_loss": 0.2805452048778534, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.5401399752984768, | |
| "grad_norm": 0.27216726541519165, | |
| "learning_rate": 2.5586064340081516e-07, | |
| "logits/chosen": 10.603267669677734, | |
| "logits/rejected": 11.226158142089844, | |
| "logps/chosen": -78.87698364257812, | |
| "logps/ref_chosen": -106.42051696777344, | |
| "logps/ref_rejected": -122.25247192382812, | |
| "logps/rejected": -94.32707977294922, | |
| "loss": 0.2862, | |
| "rewards/accuracies": 0.515625, | |
| "rewards/chosen": 0.275435209274292, | |
| "rewards/margins": -0.003818750847131014, | |
| "rewards/rejected": 0.2792539596557617, | |
| "sft_loss": 0.28622862696647644, | |
| "step": 41 | |
| }, | |
| { | |
| "epoch": 0.553314121037464, | |
| "grad_norm": 0.33358198404312134, | |
| "learning_rate": 2.4413935659918487e-07, | |
| "logits/chosen": 9.588984489440918, | |
| "logits/rejected": 10.6452054977417, | |
| "logps/chosen": -74.2744369506836, | |
| "logps/ref_chosen": -103.1148452758789, | |
| "logps/ref_rejected": -116.55464935302734, | |
| "logps/rejected": -88.39811706542969, | |
| "loss": 0.2641, | |
| "rewards/accuracies": 0.546875, | |
| "rewards/chosen": 0.28840407729148865, | |
| "rewards/margins": 0.0068388087674975395, | |
| "rewards/rejected": 0.28156527876853943, | |
| "sft_loss": 0.2640990614891052, | |
| "step": 42 | |
| }, | |
| { | |
| "epoch": 0.5664882667764513, | |
| "grad_norm": 0.47038576006889343, | |
| "learning_rate": 2.3243095276367684e-07, | |
| "logits/chosen": 9.663254737854004, | |
| "logits/rejected": 10.8170747756958, | |
| "logps/chosen": -76.7999267578125, | |
| "logps/ref_chosen": -104.21064758300781, | |
| "logps/ref_rejected": -118.7614974975586, | |
| "logps/rejected": -91.26052856445312, | |
| "loss": 0.2744, | |
| "rewards/accuracies": 0.4609375, | |
| "rewards/chosen": 0.2741071879863739, | |
| "rewards/margins": -0.0009024296887218952, | |
| "rewards/rejected": 0.2750096321105957, | |
| "sft_loss": 0.2744351625442505, | |
| "step": 43 | |
| }, | |
| { | |
| "epoch": 0.5796624125154385, | |
| "grad_norm": 0.30214762687683105, | |
| "learning_rate": 2.2076116950697937e-07, | |
| "logits/chosen": 9.781536102294922, | |
| "logits/rejected": 10.607444763183594, | |
| "logps/chosen": -72.30926513671875, | |
| "logps/ref_chosen": -100.59449005126953, | |
| "logps/ref_rejected": -115.95166778564453, | |
| "logps/rejected": -87.6107177734375, | |
| "loss": 0.2461, | |
| "rewards/accuracies": 0.5234375, | |
| "rewards/chosen": 0.28285229206085205, | |
| "rewards/margins": -0.0005572582595050335, | |
| "rewards/rejected": 0.2834095358848572, | |
| "sft_loss": 0.24606820940971375, | |
| "step": 44 | |
| }, | |
| { | |
| "epoch": 0.5928365582544257, | |
| "grad_norm": 0.26689764857292175, | |
| "learning_rate": 2.091556595453674e-07, | |
| "logits/chosen": 9.98751449584961, | |
| "logits/rejected": 10.849931716918945, | |
| "logps/chosen": -78.55535888671875, | |
| "logps/ref_chosen": -106.96060943603516, | |
| "logps/ref_rejected": -125.49449157714844, | |
| "logps/rejected": -98.11154174804688, | |
| "loss": 0.2706, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": 0.2840524911880493, | |
| "rewards/margins": 0.010223127901554108, | |
| "rewards/rejected": 0.2738293409347534, | |
| "sft_loss": 0.2706489562988281, | |
| "step": 45 | |
| }, | |
| { | |
| "epoch": 0.606010703993413, | |
| "grad_norm": 0.24426043033599854, | |
| "learning_rate": 1.9763993430846392e-07, | |
| "logits/chosen": 10.033075332641602, | |
| "logits/rejected": 10.728992462158203, | |
| "logps/chosen": -76.60491180419922, | |
| "logps/ref_chosen": -107.08544158935547, | |
| "logps/ref_rejected": -120.38542175292969, | |
| "logps/rejected": -89.13396453857422, | |
| "loss": 0.2738, | |
| "rewards/accuracies": 0.4609375, | |
| "rewards/chosen": 0.30480533838272095, | |
| "rewards/margins": -0.00770913390442729, | |
| "rewards/rejected": 0.31251445412635803, | |
| "sft_loss": 0.2737652063369751, | |
| "step": 46 | |
| }, | |
| { | |
| "epoch": 0.6191848497324002, | |
| "grad_norm": 0.22300882637500763, | |
| "learning_rate": 1.862393078595809e-07, | |
| "logits/chosen": 9.950118064880371, | |
| "logits/rejected": 11.169652938842773, | |
| "logps/chosen": -77.8197021484375, | |
| "logps/ref_chosen": -105.74787902832031, | |
| "logps/ref_rejected": -122.93606567382812, | |
| "logps/rejected": -96.1205062866211, | |
| "loss": 0.2652, | |
| "rewards/accuracies": 0.5078125, | |
| "rewards/chosen": 0.2792818248271942, | |
| "rewards/margins": 0.011126276105642319, | |
| "rewards/rejected": 0.2681555151939392, | |
| "sft_loss": 0.26522505283355713, | |
| "step": 47 | |
| }, | |
| { | |
| "epoch": 0.6323589954713874, | |
| "grad_norm": 0.2568327784538269, | |
| "learning_rate": 1.7497884124991485e-07, | |
| "logits/chosen": 10.451184272766113, | |
| "logits/rejected": 11.392836570739746, | |
| "logps/chosen": -76.05364990234375, | |
| "logps/ref_chosen": -105.3005599975586, | |
| "logps/ref_rejected": -123.93569946289062, | |
| "logps/rejected": -93.72474670410156, | |
| "loss": 0.2697, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.29246923327445984, | |
| "rewards/margins": -0.009640296921133995, | |
| "rewards/rejected": 0.3021095395088196, | |
| "sft_loss": 0.26966598629951477, | |
| "step": 48 | |
| }, | |
| { | |
| "epoch": 0.6455331412103746, | |
| "grad_norm": 0.2787468433380127, | |
| "learning_rate": 1.6388328742891678e-07, | |
| "logits/chosen": 10.48294734954834, | |
| "logits/rejected": 11.362519264221191, | |
| "logps/chosen": -74.37510681152344, | |
| "logps/ref_chosen": -104.30430603027344, | |
| "logps/ref_rejected": -115.85497283935547, | |
| "logps/rejected": -86.97918701171875, | |
| "loss": 0.2744, | |
| "rewards/accuracies": 0.46875, | |
| "rewards/chosen": 0.2992919683456421, | |
| "rewards/margins": 0.010534043423831463, | |
| "rewards/rejected": 0.28875789046287537, | |
| "sft_loss": 0.2743627727031708, | |
| "step": 49 | |
| }, | |
| { | |
| "epoch": 0.6587072869493619, | |
| "grad_norm": 0.3125001788139343, | |
| "learning_rate": 1.5297703683193753e-07, | |
| "logits/chosen": 10.078466415405273, | |
| "logits/rejected": 10.955018997192383, | |
| "logps/chosen": -75.85308837890625, | |
| "logps/ref_chosen": -104.65946960449219, | |
| "logps/ref_rejected": -118.84170532226562, | |
| "logps/rejected": -90.01569366455078, | |
| "loss": 0.2577, | |
| "rewards/accuracies": 0.4765625, | |
| "rewards/chosen": 0.28806373476982117, | |
| "rewards/margins": -0.00019637378863990307, | |
| "rewards/rejected": 0.2882601022720337, | |
| "sft_loss": 0.25767678022384644, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.6718814326883491, | |
| "grad_norm": 0.15405406057834625, | |
| "learning_rate": 1.422840637647574e-07, | |
| "logits/chosen": 10.179081916809082, | |
| "logits/rejected": 10.733241081237793, | |
| "logps/chosen": -74.97821807861328, | |
| "logps/ref_chosen": -104.4243392944336, | |
| "logps/ref_rejected": -117.16233825683594, | |
| "logps/rejected": -88.18524932861328, | |
| "loss": 0.2584, | |
| "rewards/accuracies": 0.5234375, | |
| "rewards/chosen": 0.294461190700531, | |
| "rewards/margins": 0.004690280184149742, | |
| "rewards/rejected": 0.2897709012031555, | |
| "sft_loss": 0.25844162702560425, | |
| "step": 51 | |
| }, | |
| { | |
| "epoch": 0.6850555784273363, | |
| "grad_norm": 0.16120746731758118, | |
| "learning_rate": 1.3182787370285865e-07, | |
| "logits/chosen": 9.57602596282959, | |
| "logits/rejected": 10.736429214477539, | |
| "logps/chosen": -72.85411071777344, | |
| "logps/ref_chosen": -101.99165344238281, | |
| "logps/ref_rejected": -123.20516204833984, | |
| "logps/rejected": -93.20382690429688, | |
| "loss": 0.2552, | |
| "rewards/accuracies": 0.4609375, | |
| "rewards/chosen": 0.29137539863586426, | |
| "rewards/margins": -0.008637862280011177, | |
| "rewards/rejected": 0.3000132739543915, | |
| "sft_loss": 0.2551620900630951, | |
| "step": 52 | |
| }, | |
| { | |
| "epoch": 0.6982297241663236, | |
| "grad_norm": 0.19364570081233978, | |
| "learning_rate": 1.2163145162128946e-07, | |
| "logits/chosen": 10.08875846862793, | |
| "logits/rejected": 10.98922348022461, | |
| "logps/chosen": -78.81430053710938, | |
| "logps/ref_chosen": -108.26175689697266, | |
| "logps/ref_rejected": -118.12374114990234, | |
| "logps/rejected": -89.11669921875, | |
| "loss": 0.2577, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": 0.2944745421409607, | |
| "rewards/margins": 0.004404103849083185, | |
| "rewards/rejected": 0.29007044434547424, | |
| "sft_loss": 0.2576565444469452, | |
| "step": 53 | |
| }, | |
| { | |
| "epoch": 0.7114038699053108, | |
| "grad_norm": 0.28979793190956116, | |
| "learning_rate": 1.1171721146870014e-07, | |
| "logits/chosen": 10.206223487854004, | |
| "logits/rejected": 11.192782402038574, | |
| "logps/chosen": -76.93553924560547, | |
| "logps/ref_chosen": -108.5864028930664, | |
| "logps/ref_rejected": -130.25155639648438, | |
| "logps/rejected": -96.52012634277344, | |
| "loss": 0.2871, | |
| "rewards/accuracies": 0.3984375, | |
| "rewards/chosen": 0.31650859117507935, | |
| "rewards/margins": -0.020805664360523224, | |
| "rewards/rejected": 0.337314248085022, | |
| "sft_loss": 0.2871078848838806, | |
| "step": 54 | |
| }, | |
| { | |
| "epoch": 0.724578015644298, | |
| "grad_norm": 0.1910555362701416, | |
| "learning_rate": 1.0210694689661939e-07, | |
| "logits/chosen": 10.210641860961914, | |
| "logits/rejected": 11.032613754272461, | |
| "logps/chosen": -74.83497619628906, | |
| "logps/ref_chosen": -105.69741821289062, | |
| "logps/ref_rejected": -122.07044219970703, | |
| "logps/rejected": -90.28074645996094, | |
| "loss": 0.2573, | |
| "rewards/accuracies": 0.46875, | |
| "rewards/chosen": 0.308624267578125, | |
| "rewards/margins": -0.009272638708353043, | |
| "rewards/rejected": 0.3178969621658325, | |
| "sft_loss": 0.25734585523605347, | |
| "step": 55 | |
| }, | |
| { | |
| "epoch": 0.7377521613832853, | |
| "grad_norm": 0.27673250436782837, | |
| "learning_rate": 9.282178335227883e-08, | |
| "logits/chosen": 9.825605392456055, | |
| "logits/rejected": 11.014497756958008, | |
| "logps/chosen": -76.0024642944336, | |
| "logps/ref_chosen": -106.5007095336914, | |
| "logps/ref_rejected": -123.01736450195312, | |
| "logps/rejected": -92.10081481933594, | |
| "loss": 0.2743, | |
| "rewards/accuracies": 0.5234375, | |
| "rewards/chosen": 0.3049824833869934, | |
| "rewards/margins": -0.004183035343885422, | |
| "rewards/rejected": 0.30916550755500793, | |
| "sft_loss": 0.27425283193588257, | |
| "step": 56 | |
| }, | |
| { | |
| "epoch": 0.7509263071222725, | |
| "grad_norm": 0.18975986540317535, | |
| "learning_rate": 8.388213164029459e-08, | |
| "logits/chosen": 10.642633438110352, | |
| "logits/rejected": 11.387500762939453, | |
| "logps/chosen": -79.39006805419922, | |
| "logps/ref_chosen": -109.18460083007812, | |
| "logps/ref_rejected": -124.3697280883789, | |
| "logps/rejected": -92.89251708984375, | |
| "loss": 0.2729, | |
| "rewards/accuracies": 0.4375, | |
| "rewards/chosen": 0.2979453504085541, | |
| "rewards/margins": -0.016826828941702843, | |
| "rewards/rejected": 0.3147721290588379, | |
| "sft_loss": 0.2728780210018158, | |
| "step": 57 | |
| }, | |
| { | |
| "epoch": 0.7641004528612598, | |
| "grad_norm": 0.41853123903274536, | |
| "learning_rate": 7.530764305528958e-08, | |
| "logits/chosen": 9.80534839630127, | |
| "logits/rejected": 10.46203327178955, | |
| "logps/chosen": -73.13560485839844, | |
| "logps/ref_chosen": -104.43944549560547, | |
| "logps/ref_rejected": -118.44985961914062, | |
| "logps/rejected": -85.20870208740234, | |
| "loss": 0.2597, | |
| "rewards/accuracies": 0.4765625, | |
| "rewards/chosen": 0.3130384385585785, | |
| "rewards/margins": -0.01937328279018402, | |
| "rewards/rejected": 0.33241167664527893, | |
| "sft_loss": 0.2597176730632782, | |
| "step": 58 | |
| }, | |
| { | |
| "epoch": 0.7772745986002471, | |
| "grad_norm": 0.2700183391571045, | |
| "learning_rate": 6.711716618408281e-08, | |
| "logits/chosen": 10.213776588439941, | |
| "logits/rejected": 11.164811134338379, | |
| "logps/chosen": -73.02882385253906, | |
| "logps/ref_chosen": -103.32658386230469, | |
| "logps/ref_rejected": -121.63726806640625, | |
| "logps/rejected": -90.34732818603516, | |
| "loss": 0.2687, | |
| "rewards/accuracies": 0.4296875, | |
| "rewards/chosen": 0.3029775619506836, | |
| "rewards/margins": -0.009921873919665813, | |
| "rewards/rejected": 0.3128994107246399, | |
| "sft_loss": 0.26870667934417725, | |
| "step": 59 | |
| }, | |
| { | |
| "epoch": 0.7904487443392343, | |
| "grad_norm": 0.14984337985515594, | |
| "learning_rate": 5.932870547240454e-08, | |
| "logits/chosen": 9.960476875305176, | |
| "logits/rejected": 11.033708572387695, | |
| "logps/chosen": -72.57840728759766, | |
| "logps/ref_chosen": -102.98921966552734, | |
| "logps/ref_rejected": -124.47185516357422, | |
| "logps/rejected": -92.38633728027344, | |
| "loss": 0.2563, | |
| "rewards/accuracies": 0.46875, | |
| "rewards/chosen": 0.3041080832481384, | |
| "rewards/margins": -0.01674714870750904, | |
| "rewards/rejected": 0.32085520029067993, | |
| "sft_loss": 0.2563033699989319, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.8036228900782215, | |
| "grad_norm": 0.31175732612609863, | |
| "learning_rate": 5.1959381647217665e-08, | |
| "logits/chosen": 9.977319717407227, | |
| "logits/rejected": 10.946649551391602, | |
| "logps/chosen": -73.95118713378906, | |
| "logps/ref_chosen": -106.28311157226562, | |
| "logps/ref_rejected": -121.47750854492188, | |
| "logps/rejected": -88.92882537841797, | |
| "loss": 0.2666, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.3233192563056946, | |
| "rewards/margins": -0.0021677182521671057, | |
| "rewards/rejected": 0.3254869878292084, | |
| "sft_loss": 0.26657047867774963, | |
| "step": 61 | |
| }, | |
| { | |
| "epoch": 0.8167970358172087, | |
| "grad_norm": 0.3104262351989746, | |
| "learning_rate": 4.502539408164385e-08, | |
| "logits/chosen": 10.092552185058594, | |
| "logits/rejected": 10.915599822998047, | |
| "logps/chosen": -77.04930114746094, | |
| "logps/ref_chosen": -109.67979431152344, | |
| "logps/ref_rejected": -120.36711120605469, | |
| "logps/rejected": -87.77507781982422, | |
| "loss": 0.2549, | |
| "rewards/accuracies": 0.4921875, | |
| "rewards/chosen": 0.32630497217178345, | |
| "rewards/margins": 0.00038449978455901146, | |
| "rewards/rejected": 0.3259204030036926, | |
| "sft_loss": 0.2549050450325012, | |
| "step": 62 | |
| }, | |
| { | |
| "epoch": 0.829971181556196, | |
| "grad_norm": 1.811981201171875, | |
| "learning_rate": 3.854198518522564e-08, | |
| "logits/chosen": 10.26008415222168, | |
| "logits/rejected": 11.338315963745117, | |
| "logps/chosen": -75.18155670166016, | |
| "logps/ref_chosen": -106.88896942138672, | |
| "logps/ref_rejected": -122.57796478271484, | |
| "logps/rejected": -91.81503295898438, | |
| "loss": 0.2708, | |
| "rewards/accuracies": 0.5234375, | |
| "rewards/chosen": 0.3170740604400635, | |
| "rewards/margins": 0.009444723837077618, | |
| "rewards/rejected": 0.3076293170452118, | |
| "sft_loss": 0.270816445350647, | |
| "step": 63 | |
| }, | |
| { | |
| "epoch": 0.8431453272951832, | |
| "grad_norm": 0.19537799060344696, | |
| "learning_rate": 3.2523406897802444e-08, | |
| "logits/chosen": 10.241344451904297, | |
| "logits/rejected": 10.94163990020752, | |
| "logps/chosen": -78.00033569335938, | |
| "logps/ref_chosen": -109.29510498046875, | |
| "logps/ref_rejected": -121.51821899414062, | |
| "logps/rejected": -88.49795532226562, | |
| "loss": 0.2733, | |
| "rewards/accuracies": 0.40625, | |
| "rewards/chosen": 0.3129475712776184, | |
| "rewards/margins": -0.017255008220672607, | |
| "rewards/rejected": 0.33020254969596863, | |
| "sft_loss": 0.2733091115951538, | |
| "step": 64 | |
| }, | |
| { | |
| "epoch": 0.8563194730341704, | |
| "grad_norm": 0.3109639286994934, | |
| "learning_rate": 2.6982889360653376e-08, | |
| "logits/chosen": 9.754749298095703, | |
| "logits/rejected": 11.062653541564941, | |
| "logps/chosen": -76.51459503173828, | |
| "logps/ref_chosen": -110.15232849121094, | |
| "logps/ref_rejected": -127.54279327392578, | |
| "logps/rejected": -96.31084442138672, | |
| "loss": 0.271, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": 0.3363773226737976, | |
| "rewards/margins": 0.024057911708950996, | |
| "rewards/rejected": 0.31231939792633057, | |
| "sft_loss": 0.27095291018486023, | |
| "step": 65 | |
| }, | |
| { | |
| "epoch": 0.8694936187731577, | |
| "grad_norm": 0.22148017585277557, | |
| "learning_rate": 2.1932611833775843e-08, | |
| "logits/chosen": 9.787257194519043, | |
| "logits/rejected": 10.842453956604004, | |
| "logps/chosen": -72.53459167480469, | |
| "logps/ref_chosen": -104.0207748413086, | |
| "logps/ref_rejected": -126.93211364746094, | |
| "logps/rejected": -92.82463073730469, | |
| "loss": 0.2518, | |
| "rewards/accuracies": 0.4453125, | |
| "rewards/chosen": 0.31486180424690247, | |
| "rewards/margins": -0.026213109493255615, | |
| "rewards/rejected": 0.3410749137401581, | |
| "sft_loss": 0.25176769495010376, | |
| "step": 66 | |
| }, | |
| { | |
| "epoch": 0.8826677645121449, | |
| "grad_norm": 0.23339781165122986, | |
| "learning_rate": 1.738367592322837e-08, | |
| "logits/chosen": 10.151799201965332, | |
| "logits/rejected": 11.080910682678223, | |
| "logps/chosen": -72.54059600830078, | |
| "logps/ref_chosen": -104.55751037597656, | |
| "logps/ref_rejected": -119.71514892578125, | |
| "logps/rejected": -87.53738403320312, | |
| "loss": 0.2572, | |
| "rewards/accuracies": 0.484375, | |
| "rewards/chosen": 0.32016903162002563, | |
| "rewards/margins": -0.001608673483133316, | |
| "rewards/rejected": 0.3217777609825134, | |
| "sft_loss": 0.25715309381484985, | |
| "step": 67 | |
| }, | |
| { | |
| "epoch": 0.8958419102511321, | |
| "grad_norm": 0.3316951394081116, | |
| "learning_rate": 1.3346081177391472e-08, | |
| "logits/chosen": 10.458593368530273, | |
| "logits/rejected": 10.821167945861816, | |
| "logps/chosen": -76.2717056274414, | |
| "logps/ref_chosen": -107.26033020019531, | |
| "logps/ref_rejected": -115.8590087890625, | |
| "logps/rejected": -83.77940368652344, | |
| "loss": 0.2695, | |
| "rewards/accuracies": 0.4609375, | |
| "rewards/chosen": 0.3098861873149872, | |
| "rewards/margins": -0.010909780859947205, | |
| "rewards/rejected": 0.3207959830760956, | |
| "sft_loss": 0.26949527859687805, | |
| "step": 68 | |
| }, | |
| { | |
| "epoch": 0.9090160559901194, | |
| "grad_norm": 0.5555047392845154, | |
| "learning_rate": 9.828703105789981e-09, | |
| "logits/chosen": 10.413060188293457, | |
| "logits/rejected": 11.321882247924805, | |
| "logps/chosen": -73.33889770507812, | |
| "logps/ref_chosen": -106.8610610961914, | |
| "logps/ref_rejected": -122.44428253173828, | |
| "logps/rejected": -89.23860931396484, | |
| "loss": 0.2688, | |
| "rewards/accuracies": 0.5078125, | |
| "rewards/chosen": 0.3352215886116028, | |
| "rewards/margins": 0.003164912573993206, | |
| "rewards/rejected": 0.3320567011833191, | |
| "sft_loss": 0.26880374550819397, | |
| "step": 69 | |
| }, | |
| { | |
| "epoch": 0.9221902017291066, | |
| "grad_norm": 0.31241339445114136, | |
| "learning_rate": 6.839273668796747e-09, | |
| "logits/chosen": 10.117363929748535, | |
| "logits/rejected": 10.65896224975586, | |
| "logps/chosen": -76.24555969238281, | |
| "logps/ref_chosen": -106.58778381347656, | |
| "logps/ref_rejected": -124.77790832519531, | |
| "logps/rejected": -93.02002716064453, | |
| "loss": 0.2603, | |
| "rewards/accuracies": 0.4453125, | |
| "rewards/chosen": 0.3034222424030304, | |
| "rewards/margins": -0.014156593009829521, | |
| "rewards/rejected": 0.3175787925720215, | |
| "sft_loss": 0.2603415846824646, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.9353643474680938, | |
| "grad_norm": 0.3154677748680115, | |
| "learning_rate": 4.384364281105973e-09, | |
| "logits/chosen": 10.104023933410645, | |
| "logits/rejected": 11.347376823425293, | |
| "logps/chosen": -70.88841247558594, | |
| "logps/ref_chosen": -104.39148712158203, | |
| "logps/ref_rejected": -120.59461212158203, | |
| "logps/rejected": -87.24669647216797, | |
| "loss": 0.2746, | |
| "rewards/accuracies": 0.4921875, | |
| "rewards/chosen": 0.33503076434135437, | |
| "rewards/margins": 0.001551617868244648, | |
| "rewards/rejected": 0.3334791362285614, | |
| "sft_loss": 0.2746419608592987, | |
| "step": 71 | |
| }, | |
| { | |
| "epoch": 0.9485384932070811, | |
| "grad_norm": 0.27243080735206604, | |
| "learning_rate": 2.469371366337264e-09, | |
| "logits/chosen": 10.178709030151367, | |
| "logits/rejected": 11.207265853881836, | |
| "logps/chosen": -77.24491882324219, | |
| "logps/ref_chosen": -108.53898620605469, | |
| "logps/ref_rejected": -123.26167297363281, | |
| "logps/rejected": -91.20188903808594, | |
| "loss": 0.2649, | |
| "rewards/accuracies": 0.4765625, | |
| "rewards/chosen": 0.31294065713882446, | |
| "rewards/margins": -0.007657179608941078, | |
| "rewards/rejected": 0.3205978274345398, | |
| "sft_loss": 0.26487135887145996, | |
| "step": 72 | |
| }, | |
| { | |
| "epoch": 0.9617126389460683, | |
| "grad_norm": 0.29381245374679565, | |
| "learning_rate": 1.0985044945254762e-09, | |
| "logits/chosen": 10.011255264282227, | |
| "logits/rejected": 10.857406616210938, | |
| "logps/chosen": -79.84062957763672, | |
| "logps/ref_chosen": -110.50595092773438, | |
| "logps/ref_rejected": -124.92510223388672, | |
| "logps/rejected": -94.2270736694336, | |
| "loss": 0.2583, | |
| "rewards/accuracies": 0.4921875, | |
| "rewards/chosen": 0.3066532015800476, | |
| "rewards/margins": -0.0003270732704550028, | |
| "rewards/rejected": 0.30698028206825256, | |
| "sft_loss": 0.258339524269104, | |
| "step": 73 | |
| }, | |
| { | |
| "epoch": 0.9748867846850556, | |
| "grad_norm": 0.4308127462863922, | |
| "learning_rate": 2.7477712857215675e-10, | |
| "logits/chosen": 10.271313667297363, | |
| "logits/rejected": 11.117928504943848, | |
| "logps/chosen": -75.68081665039062, | |
| "logps/ref_chosen": -107.1200942993164, | |
| "logps/ref_rejected": -120.22421264648438, | |
| "logps/rejected": -89.6799545288086, | |
| "loss": 0.2564, | |
| "rewards/accuracies": 0.546875, | |
| "rewards/chosen": 0.3143928647041321, | |
| "rewards/margins": 0.008950251154601574, | |
| "rewards/rejected": 0.3054426312446594, | |
| "sft_loss": 0.2563988268375397, | |
| "step": 74 | |
| }, | |
| { | |
| "epoch": 0.9880609304240429, | |
| "grad_norm": 0.1982087343931198, | |
| "learning_rate": 0.0, | |
| "logits/chosen": 10.055291175842285, | |
| "logits/rejected": 10.858556747436523, | |
| "logps/chosen": -75.02184295654297, | |
| "logps/ref_chosen": -107.40764617919922, | |
| "logps/ref_rejected": -120.6578369140625, | |
| "logps/rejected": -89.03117370605469, | |
| "loss": 0.2485, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": 0.32385802268981934, | |
| "rewards/margins": 0.0075914738699793816, | |
| "rewards/rejected": 0.3162665367126465, | |
| "sft_loss": 0.24852335453033447, | |
| "step": 75 | |
| }, | |
| { | |
| "epoch": 0.9880609304240429, | |
| "step": 75, | |
| "total_flos": 0.0, | |
| "train_loss": 0.29937881012757617, | |
| "train_runtime": 8120.0447, | |
| "train_samples_per_second": 1.196, | |
| "train_steps_per_second": 0.009 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 75, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 12, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 2, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |