| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.9880609304240429, | |
| "eval_steps": 500, | |
| "global_step": 75, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.013174145738987238, | |
| "grad_norm": 0.8317140340805054, | |
| "learning_rate": 6.25e-08, | |
| "logits/chosen": 9.990612030029297, | |
| "logits/rejected": 10.698101997375488, | |
| "logps/chosen": -102.88545989990234, | |
| "logps/ref_chosen": -102.88545989990234, | |
| "logps/ref_rejected": -121.84871673583984, | |
| "logps/rejected": -121.84871673583984, | |
| "loss": 0.4978, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "sft_loss": 0.36753880977630615, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.026348291477974475, | |
| "grad_norm": 0.2926611006259918, | |
| "learning_rate": 1.25e-07, | |
| "logits/chosen": 10.211905479431152, | |
| "logits/rejected": 11.06594467163086, | |
| "logps/chosen": -107.70349884033203, | |
| "logps/ref_chosen": -107.70349884033203, | |
| "logps/ref_rejected": -121.89966583251953, | |
| "logps/rejected": -121.89966583251953, | |
| "loss": 0.5233, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "sft_loss": 0.41013145446777344, | |
| "step": 2 | |
| }, | |
| { | |
| "epoch": 0.03952243721696171, | |
| "grad_norm": 0.3214672803878784, | |
| "learning_rate": 1.875e-07, | |
| "logits/chosen": 10.036933898925781, | |
| "logits/rejected": 11.024795532226562, | |
| "logps/chosen": -108.28660583496094, | |
| "logps/ref_chosen": -107.98188781738281, | |
| "logps/ref_rejected": -124.51527404785156, | |
| "logps/rejected": -124.7075424194336, | |
| "loss": 0.5249, | |
| "rewards/accuracies": 0.4921875, | |
| "rewards/chosen": -0.0030471612699329853, | |
| "rewards/margins": -0.001124453847296536, | |
| "rewards/rejected": -0.0019227075390517712, | |
| "sft_loss": 0.4123927652835846, | |
| "step": 3 | |
| }, | |
| { | |
| "epoch": 0.05269658295594895, | |
| "grad_norm": 0.7239967584609985, | |
| "learning_rate": 2.5e-07, | |
| "logits/chosen": 9.835367202758789, | |
| "logits/rejected": 10.854362487792969, | |
| "logps/chosen": -109.89208221435547, | |
| "logps/ref_chosen": -109.20836639404297, | |
| "logps/ref_rejected": -119.23908996582031, | |
| "logps/rejected": -119.69357299804688, | |
| "loss": 0.5208, | |
| "rewards/accuracies": 0.453125, | |
| "rewards/chosen": -0.006837163120508194, | |
| "rewards/margins": -0.0022922407370060682, | |
| "rewards/rejected": -0.004544922150671482, | |
| "sft_loss": 0.4051341712474823, | |
| "step": 4 | |
| }, | |
| { | |
| "epoch": 0.06587072869493618, | |
| "grad_norm": 0.9211171269416809, | |
| "learning_rate": 3.1249999999999997e-07, | |
| "logits/chosen": 10.207865715026855, | |
| "logits/rejected": 10.963621139526367, | |
| "logps/chosen": -104.02088928222656, | |
| "logps/ref_chosen": -103.87680053710938, | |
| "logps/ref_rejected": -118.41618347167969, | |
| "logps/rejected": -118.34123992919922, | |
| "loss": 0.5001, | |
| "rewards/accuracies": 0.46875, | |
| "rewards/chosen": -0.0014409449649974704, | |
| "rewards/margins": -0.002190487692132592, | |
| "rewards/rejected": 0.0007495426689274609, | |
| "sft_loss": 0.3706103563308716, | |
| "step": 5 | |
| }, | |
| { | |
| "epoch": 0.07904487443392343, | |
| "grad_norm": 0.7005686163902283, | |
| "learning_rate": 3.75e-07, | |
| "logits/chosen": 10.686239242553711, | |
| "logits/rejected": 11.462547302246094, | |
| "logps/chosen": -108.1670913696289, | |
| "logps/ref_chosen": -107.58968353271484, | |
| "logps/ref_rejected": -122.07303619384766, | |
| "logps/rejected": -122.58065032958984, | |
| "loss": 0.5134, | |
| "rewards/accuracies": 0.484375, | |
| "rewards/chosen": -0.005774094723165035, | |
| "rewards/margins": -0.0006979470490477979, | |
| "rewards/rejected": -0.005076148081570864, | |
| "sft_loss": 0.39328432083129883, | |
| "step": 6 | |
| }, | |
| { | |
| "epoch": 0.09221902017291066, | |
| "grad_norm": 0.8737779259681702, | |
| "learning_rate": 4.375e-07, | |
| "logits/chosen": 10.040006637573242, | |
| "logits/rejected": 10.747206687927246, | |
| "logps/chosen": -107.037353515625, | |
| "logps/ref_chosen": -107.42727661132812, | |
| "logps/ref_rejected": -116.87063598632812, | |
| "logps/rejected": -116.42378997802734, | |
| "loss": 0.5022, | |
| "rewards/accuracies": 0.4765625, | |
| "rewards/chosen": 0.0038992553018033504, | |
| "rewards/margins": -0.0005691752885468304, | |
| "rewards/rejected": 0.004468431230634451, | |
| "sft_loss": 0.37478095293045044, | |
| "step": 7 | |
| }, | |
| { | |
| "epoch": 0.1053931659118979, | |
| "grad_norm": 0.46314188838005066, | |
| "learning_rate": 5e-07, | |
| "logits/chosen": 10.196634292602539, | |
| "logits/rejected": 11.089151382446289, | |
| "logps/chosen": -104.9737548828125, | |
| "logps/ref_chosen": -105.60282135009766, | |
| "logps/ref_rejected": -119.53916931152344, | |
| "logps/rejected": -118.9640884399414, | |
| "loss": 0.5047, | |
| "rewards/accuracies": 0.4921875, | |
| "rewards/chosen": 0.006290654186159372, | |
| "rewards/margins": 0.0005399012006819248, | |
| "rewards/rejected": 0.0057507529854774475, | |
| "sft_loss": 0.37916287779808044, | |
| "step": 8 | |
| }, | |
| { | |
| "epoch": 0.11856731165088513, | |
| "grad_norm": 0.6739105582237244, | |
| "learning_rate": 4.997252228714278e-07, | |
| "logits/chosen": 10.174893379211426, | |
| "logits/rejected": 11.141225814819336, | |
| "logps/chosen": -104.830078125, | |
| "logps/ref_chosen": -105.46086120605469, | |
| "logps/ref_rejected": -119.00373840332031, | |
| "logps/rejected": -118.41084289550781, | |
| "loss": 0.5073, | |
| "rewards/accuracies": 0.546875, | |
| "rewards/chosen": 0.006307927425950766, | |
| "rewards/margins": 0.00037892413092777133, | |
| "rewards/rejected": 0.005929003469645977, | |
| "sft_loss": 0.3834277391433716, | |
| "step": 9 | |
| }, | |
| { | |
| "epoch": 0.13174145738987236, | |
| "grad_norm": 0.32560959458351135, | |
| "learning_rate": 4.989014955054745e-07, | |
| "logits/chosen": 10.061932563781738, | |
| "logits/rejected": 10.884778022766113, | |
| "logps/chosen": -102.10572052001953, | |
| "logps/ref_chosen": -104.21009826660156, | |
| "logps/ref_rejected": -118.9209213256836, | |
| "logps/rejected": -117.1087646484375, | |
| "loss": 0.4814, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": 0.02104363776743412, | |
| "rewards/margins": 0.00292214541696012, | |
| "rewards/rejected": 0.018121493980288506, | |
| "sft_loss": 0.34126415848731995, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.14491560312885962, | |
| "grad_norm": 0.8538271188735962, | |
| "learning_rate": 4.975306286336627e-07, | |
| "logits/chosen": 9.989534378051758, | |
| "logits/rejected": 11.177312850952148, | |
| "logps/chosen": -102.71998596191406, | |
| "logps/ref_chosen": -105.94319152832031, | |
| "logps/ref_rejected": -122.76007843017578, | |
| "logps/rejected": -119.8812026977539, | |
| "loss": 0.5112, | |
| "rewards/accuracies": 0.5078125, | |
| "rewards/chosen": 0.03223201259970665, | |
| "rewards/margins": 0.0034431489184498787, | |
| "rewards/rejected": 0.028788862749934196, | |
| "sft_loss": 0.3909299075603485, | |
| "step": 11 | |
| }, | |
| { | |
| "epoch": 0.15808974886784685, | |
| "grad_norm": 0.599084734916687, | |
| "learning_rate": 4.956156357188939e-07, | |
| "logits/chosen": 9.907474517822266, | |
| "logits/rejected": 10.599059104919434, | |
| "logps/chosen": -105.25791931152344, | |
| "logps/ref_chosen": -109.08442687988281, | |
| "logps/ref_rejected": -121.41947174072266, | |
| "logps/rejected": -117.7214584350586, | |
| "loss": 0.4934, | |
| "rewards/accuracies": 0.5546875, | |
| "rewards/chosen": 0.03826504573225975, | |
| "rewards/margins": 0.00128496577963233, | |
| "rewards/rejected": 0.036980077624320984, | |
| "sft_loss": 0.3606122136116028, | |
| "step": 12 | |
| }, | |
| { | |
| "epoch": 0.17126389460683408, | |
| "grad_norm": 0.20007266104221344, | |
| "learning_rate": 4.931607263312032e-07, | |
| "logits/chosen": 9.952820777893066, | |
| "logits/rejected": 11.027226448059082, | |
| "logps/chosen": -99.99949645996094, | |
| "logps/ref_chosen": -104.62150573730469, | |
| "logps/ref_rejected": -119.55384063720703, | |
| "logps/rejected": -115.20744323730469, | |
| "loss": 0.5005, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": 0.04622000828385353, | |
| "rewards/margins": 0.0027560186572372913, | |
| "rewards/rejected": 0.04346399009227753, | |
| "sft_loss": 0.37296974658966064, | |
| "step": 13 | |
| }, | |
| { | |
| "epoch": 0.1844380403458213, | |
| "grad_norm": 0.30717214941978455, | |
| "learning_rate": 4.9017129689421e-07, | |
| "logits/chosen": 10.493995666503906, | |
| "logits/rejected": 11.61634635925293, | |
| "logps/chosen": -100.9910888671875, | |
| "logps/ref_chosen": -106.179443359375, | |
| "logps/ref_rejected": -120.73036193847656, | |
| "logps/rejected": -115.51315307617188, | |
| "loss": 0.4904, | |
| "rewards/accuracies": 0.484375, | |
| "rewards/chosen": 0.051883526146411896, | |
| "rewards/margins": -0.0002885278081521392, | |
| "rewards/rejected": 0.05217204988002777, | |
| "sft_loss": 0.35513877868652344, | |
| "step": 14 | |
| }, | |
| { | |
| "epoch": 0.19761218608480857, | |
| "grad_norm": 0.31972214579582214, | |
| "learning_rate": 4.866539188226085e-07, | |
| "logits/chosen": 9.85748291015625, | |
| "logits/rejected": 10.79136848449707, | |
| "logps/chosen": -98.44343566894531, | |
| "logps/ref_chosen": -105.70547485351562, | |
| "logps/ref_rejected": -118.89997863769531, | |
| "logps/rejected": -111.49993133544922, | |
| "loss": 0.4836, | |
| "rewards/accuracies": 0.546875, | |
| "rewards/chosen": 0.0726204589009285, | |
| "rewards/margins": -0.001380045199766755, | |
| "rewards/rejected": 0.07400050014257431, | |
| "sft_loss": 0.3432847261428833, | |
| "step": 15 | |
| }, | |
| { | |
| "epoch": 0.2107863318237958, | |
| "grad_norm": 0.3299219012260437, | |
| "learning_rate": 4.826163240767716e-07, | |
| "logits/chosen": 10.693929672241211, | |
| "logits/rejected": 11.317272186279297, | |
| "logps/chosen": -99.89289855957031, | |
| "logps/ref_chosen": -108.86376953125, | |
| "logps/ref_rejected": -122.1635513305664, | |
| "logps/rejected": -113.75732421875, | |
| "loss": 0.4939, | |
| "rewards/accuracies": 0.578125, | |
| "rewards/chosen": 0.08970852196216583, | |
| "rewards/margins": 0.005646157078444958, | |
| "rewards/rejected": 0.084062360227108, | |
| "sft_loss": 0.3626875579357147, | |
| "step": 16 | |
| }, | |
| { | |
| "epoch": 0.22396047756278303, | |
| "grad_norm": 0.39949116110801697, | |
| "learning_rate": 4.780673881662242e-07, | |
| "logits/chosen": 10.215154647827148, | |
| "logits/rejected": 10.865469932556152, | |
| "logps/chosen": -92.98998260498047, | |
| "logps/ref_chosen": -102.93986511230469, | |
| "logps/ref_rejected": -119.43718719482422, | |
| "logps/rejected": -109.6163558959961, | |
| "loss": 0.4998, | |
| "rewards/accuracies": 0.4375, | |
| "rewards/chosen": 0.09949885308742523, | |
| "rewards/margins": 0.0012904854957014322, | |
| "rewards/rejected": 0.09820836037397385, | |
| "sft_loss": 0.37107497453689575, | |
| "step": 17 | |
| }, | |
| { | |
| "epoch": 0.23713462330177026, | |
| "grad_norm": 0.3356131911277771, | |
| "learning_rate": 4.730171106393466e-07, | |
| "logits/chosen": 10.427769660949707, | |
| "logits/rejected": 11.224242210388184, | |
| "logps/chosen": -92.83369445800781, | |
| "logps/ref_chosen": -103.81341552734375, | |
| "logps/ref_rejected": -117.45123291015625, | |
| "logps/rejected": -107.25675964355469, | |
| "loss": 0.485, | |
| "rewards/accuracies": 0.5078125, | |
| "rewards/chosen": 0.10979717969894409, | |
| "rewards/margins": 0.007852486334741116, | |
| "rewards/rejected": 0.101944699883461, | |
| "sft_loss": 0.3487062454223633, | |
| "step": 18 | |
| }, | |
| { | |
| "epoch": 0.2503087690407575, | |
| "grad_norm": 0.31761711835861206, | |
| "learning_rate": 4.6747659310219757e-07, | |
| "logits/chosen": 10.344837188720703, | |
| "logits/rejected": 11.015111923217773, | |
| "logps/chosen": -97.11812591552734, | |
| "logps/ref_chosen": -107.85797119140625, | |
| "logps/ref_rejected": -121.88042449951172, | |
| "logps/rejected": -110.71996307373047, | |
| "loss": 0.4774, | |
| "rewards/accuracies": 0.4609375, | |
| "rewards/chosen": 0.10739848017692566, | |
| "rewards/margins": -0.004206114914268255, | |
| "rewards/rejected": 0.11160460114479065, | |
| "sft_loss": 0.33181920647621155, | |
| "step": 19 | |
| }, | |
| { | |
| "epoch": 0.2634829147797447, | |
| "grad_norm": 0.3374970555305481, | |
| "learning_rate": 4.6145801481477433e-07, | |
| "logits/chosen": 10.763664245605469, | |
| "logits/rejected": 11.578323364257812, | |
| "logps/chosen": -92.1986083984375, | |
| "logps/ref_chosen": -103.42721557617188, | |
| "logps/ref_rejected": -116.7796630859375, | |
| "logps/rejected": -106.10646057128906, | |
| "loss": 0.4709, | |
| "rewards/accuracies": 0.5703125, | |
| "rewards/chosen": 0.11228612810373306, | |
| "rewards/margins": 0.005554294213652611, | |
| "rewards/rejected": 0.1067318320274353, | |
| "sft_loss": 0.324346125125885, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.276657060518732, | |
| "grad_norm": 0.20886239409446716, | |
| "learning_rate": 4.549746059183561e-07, | |
| "logits/chosen": 9.804973602294922, | |
| "logits/rejected": 10.907769203186035, | |
| "logps/chosen": -94.7210693359375, | |
| "logps/ref_chosen": -106.60163879394531, | |
| "logps/ref_rejected": -124.56562805175781, | |
| "logps/rejected": -112.27490234375, | |
| "loss": 0.4698, | |
| "rewards/accuracies": 0.4921875, | |
| "rewards/chosen": 0.11880576610565186, | |
| "rewards/margins": -0.004101429134607315, | |
| "rewards/rejected": 0.12290719151496887, | |
| "sft_loss": 0.3191758394241333, | |
| "step": 21 | |
| }, | |
| { | |
| "epoch": 0.28983120625771924, | |
| "grad_norm": 0.4104432165622711, | |
| "learning_rate": 4.480406183527823e-07, | |
| "logits/chosen": 10.239618301391602, | |
| "logits/rejected": 11.115823745727539, | |
| "logps/chosen": -90.41361999511719, | |
| "logps/ref_chosen": -103.77696228027344, | |
| "logps/ref_rejected": -118.73616027832031, | |
| "logps/rejected": -106.71508026123047, | |
| "loss": 0.4799, | |
| "rewards/accuracies": 0.5703125, | |
| "rewards/chosen": 0.13363340497016907, | |
| "rewards/margins": 0.013422610238194466, | |
| "rewards/rejected": 0.12021078914403915, | |
| "sft_loss": 0.34187009930610657, | |
| "step": 22 | |
| }, | |
| { | |
| "epoch": 0.3030053519967065, | |
| "grad_norm": 0.18107129633426666, | |
| "learning_rate": 4.4067129452759546e-07, | |
| "logits/chosen": 10.109177589416504, | |
| "logits/rejected": 11.132944107055664, | |
| "logps/chosen": -91.18827819824219, | |
| "logps/ref_chosen": -104.72956085205078, | |
| "logps/ref_rejected": -121.35556030273438, | |
| "logps/rejected": -108.52244567871094, | |
| "loss": 0.4793, | |
| "rewards/accuracies": 0.515625, | |
| "rewards/chosen": 0.13541287183761597, | |
| "rewards/margins": 0.007081696763634682, | |
| "rewards/rejected": 0.12833118438720703, | |
| "sft_loss": 0.33867964148521423, | |
| "step": 23 | |
| }, | |
| { | |
| "epoch": 0.3161794977356937, | |
| "grad_norm": 0.2613002359867096, | |
| "learning_rate": 4.3288283381591725e-07, | |
| "logits/chosen": 10.174164772033691, | |
| "logits/rejected": 11.02208137512207, | |
| "logps/chosen": -92.25580596923828, | |
| "logps/ref_chosen": -105.88758087158203, | |
| "logps/ref_rejected": -125.69054412841797, | |
| "logps/rejected": -112.1562728881836, | |
| "loss": 0.4738, | |
| "rewards/accuracies": 0.515625, | |
| "rewards/chosen": 0.13631772994995117, | |
| "rewards/margins": 0.0009749364107847214, | |
| "rewards/rejected": 0.1353427916765213, | |
| "sft_loss": 0.32745954394340515, | |
| "step": 24 | |
| }, | |
| { | |
| "epoch": 0.32935364347468093, | |
| "grad_norm": 0.2633656859397888, | |
| "learning_rate": 4.246923569447104e-07, | |
| "logits/chosen": 10.34875202178955, | |
| "logits/rejected": 11.083490371704102, | |
| "logps/chosen": -94.46806335449219, | |
| "logps/ref_chosen": -110.0761489868164, | |
| "logps/ref_rejected": -129.10540771484375, | |
| "logps/rejected": -113.79875183105469, | |
| "loss": 0.4713, | |
| "rewards/accuracies": 0.5546875, | |
| "rewards/chosen": 0.15608078241348267, | |
| "rewards/margins": 0.003014356829226017, | |
| "rewards/rejected": 0.15306642651557922, | |
| "sft_loss": 0.3239065706729889, | |
| "step": 25 | |
| }, | |
| { | |
| "epoch": 0.34252778921366817, | |
| "grad_norm": 0.32009434700012207, | |
| "learning_rate": 4.161178683597054e-07, | |
| "logits/chosen": 10.391422271728516, | |
| "logits/rejected": 11.489303588867188, | |
| "logps/chosen": -87.06050109863281, | |
| "logps/ref_chosen": -103.74571990966797, | |
| "logps/ref_rejected": -120.73832702636719, | |
| "logps/rejected": -104.55479431152344, | |
| "loss": 0.4641, | |
| "rewards/accuracies": 0.546875, | |
| "rewards/chosen": 0.16685229539871216, | |
| "rewards/margins": 0.005017024464905262, | |
| "rewards/rejected": 0.16183526813983917, | |
| "sft_loss": 0.3127303719520569, | |
| "step": 26 | |
| }, | |
| { | |
| "epoch": 0.3557019349526554, | |
| "grad_norm": 0.45031648874282837, | |
| "learning_rate": 4.0717821664772124e-07, | |
| "logits/chosen": 10.074949264526367, | |
| "logits/rejected": 11.320752143859863, | |
| "logps/chosen": -87.53924560546875, | |
| "logps/ref_chosen": -105.47428131103516, | |
| "logps/ref_rejected": -120.5193099975586, | |
| "logps/rejected": -103.33158111572266, | |
| "loss": 0.4803, | |
| "rewards/accuracies": 0.546875, | |
| "rewards/chosen": 0.1793503314256668, | |
| "rewards/margins": 0.0074730804190039635, | |
| "rewards/rejected": 0.17187726497650146, | |
| "sft_loss": 0.34031108021736145, | |
| "step": 27 | |
| }, | |
| { | |
| "epoch": 0.3688760806916426, | |
| "grad_norm": 0.1718786656856537, | |
| "learning_rate": 3.978930531033806e-07, | |
| "logits/chosen": 9.691258430480957, | |
| "logits/rejected": 10.853616714477539, | |
| "logps/chosen": -86.0871810913086, | |
| "logps/ref_chosen": -103.72540283203125, | |
| "logps/ref_rejected": -119.79557800292969, | |
| "logps/rejected": -102.32144927978516, | |
| "loss": 0.4547, | |
| "rewards/accuracies": 0.484375, | |
| "rewards/chosen": 0.1763821244239807, | |
| "rewards/margins": 0.0016407554503530264, | |
| "rewards/rejected": 0.17474135756492615, | |
| "sft_loss": 0.2956213057041168, | |
| "step": 28 | |
| }, | |
| { | |
| "epoch": 0.3820502264306299, | |
| "grad_norm": 0.22632478177547455, | |
| "learning_rate": 3.882827885312998e-07, | |
| "logits/chosen": 10.157691955566406, | |
| "logits/rejected": 11.235154151916504, | |
| "logps/chosen": -90.04312133789062, | |
| "logps/ref_chosen": -108.65434265136719, | |
| "logps/ref_rejected": -121.46784973144531, | |
| "logps/rejected": -103.93795776367188, | |
| "loss": 0.4546, | |
| "rewards/accuracies": 0.546875, | |
| "rewards/chosen": 0.1861121654510498, | |
| "rewards/margins": 0.010813241824507713, | |
| "rewards/rejected": 0.17529892921447754, | |
| "sft_loss": 0.29843831062316895, | |
| "step": 29 | |
| }, | |
| { | |
| "epoch": 0.39522437216961714, | |
| "grad_norm": 0.1572926938533783, | |
| "learning_rate": 3.7836854837871044e-07, | |
| "logits/chosen": 10.309174537658691, | |
| "logits/rejected": 11.701813697814941, | |
| "logps/chosen": -83.15330505371094, | |
| "logps/ref_chosen": -103.62174224853516, | |
| "logps/ref_rejected": -126.73807525634766, | |
| "logps/rejected": -107.76830291748047, | |
| "loss": 0.4646, | |
| "rewards/accuracies": 0.6015625, | |
| "rewards/chosen": 0.20468442142009735, | |
| "rewards/margins": 0.014986753463745117, | |
| "rewards/rejected": 0.18969768285751343, | |
| "sft_loss": 0.31653928756713867, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.4083985179086044, | |
| "grad_norm": 0.2045690417289734, | |
| "learning_rate": 3.681721262971413e-07, | |
| "logits/chosen": 9.92952823638916, | |
| "logits/rejected": 10.956416130065918, | |
| "logps/chosen": -85.6856689453125, | |
| "logps/ref_chosen": -106.10479736328125, | |
| "logps/ref_rejected": -120.6382827758789, | |
| "logps/rejected": -101.52389526367188, | |
| "loss": 0.4657, | |
| "rewards/accuracies": 0.515625, | |
| "rewards/chosen": 0.20419135689735413, | |
| "rewards/margins": 0.013047425076365471, | |
| "rewards/rejected": 0.1911439299583435, | |
| "sft_loss": 0.3177209496498108, | |
| "step": 31 | |
| }, | |
| { | |
| "epoch": 0.4215726636475916, | |
| "grad_norm": 0.17902837693691254, | |
| "learning_rate": 3.577159362352426e-07, | |
| "logits/chosen": 10.117594718933105, | |
| "logits/rejected": 11.502117156982422, | |
| "logps/chosen": -86.69911193847656, | |
| "logps/ref_chosen": -105.99569702148438, | |
| "logps/ref_rejected": -128.34303283691406, | |
| "logps/rejected": -108.87689208984375, | |
| "loss": 0.4587, | |
| "rewards/accuracies": 0.5234375, | |
| "rewards/chosen": 0.19296588003635406, | |
| "rewards/margins": -0.0016955481842160225, | |
| "rewards/rejected": 0.19466140866279602, | |
| "sft_loss": 0.30118295550346375, | |
| "step": 32 | |
| }, | |
| { | |
| "epoch": 0.43474680938657884, | |
| "grad_norm": 0.2392444610595703, | |
| "learning_rate": 3.470229631680624e-07, | |
| "logits/chosen": 10.104952812194824, | |
| "logits/rejected": 10.932075500488281, | |
| "logps/chosen": -85.76262664794922, | |
| "logps/ref_chosen": -105.72196197509766, | |
| "logps/ref_rejected": -121.59507751464844, | |
| "logps/rejected": -101.98007202148438, | |
| "loss": 0.4468, | |
| "rewards/accuracies": 0.515625, | |
| "rewards/chosen": 0.19959335029125214, | |
| "rewards/margins": 0.0034433496184647083, | |
| "rewards/rejected": 0.196150004863739, | |
| "sft_loss": 0.28305673599243164, | |
| "step": 33 | |
| }, | |
| { | |
| "epoch": 0.44792095512556607, | |
| "grad_norm": 0.24395713210105896, | |
| "learning_rate": 3.361167125710832e-07, | |
| "logits/chosen": 10.26183032989502, | |
| "logits/rejected": 11.106597900390625, | |
| "logps/chosen": -90.0468978881836, | |
| "logps/ref_chosen": -111.4834976196289, | |
| "logps/ref_rejected": -130.48089599609375, | |
| "logps/rejected": -108.98446655273438, | |
| "loss": 0.4699, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.21436603367328644, | |
| "rewards/margins": -0.0005981822032481432, | |
| "rewards/rejected": 0.21496421098709106, | |
| "sft_loss": 0.3201817572116852, | |
| "step": 34 | |
| }, | |
| { | |
| "epoch": 0.4610951008645533, | |
| "grad_norm": 0.28230205178260803, | |
| "learning_rate": 3.2502115875008516e-07, | |
| "logits/chosen": 10.544075012207031, | |
| "logits/rejected": 11.514993667602539, | |
| "logps/chosen": -86.92802429199219, | |
| "logps/ref_chosen": -108.9183349609375, | |
| "logps/ref_rejected": -121.32493591308594, | |
| "logps/rejected": -100.38153839111328, | |
| "loss": 0.4497, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": 0.21990321576595306, | |
| "rewards/margins": 0.010469252243638039, | |
| "rewards/rejected": 0.20943395793437958, | |
| "sft_loss": 0.29018867015838623, | |
| "step": 35 | |
| }, | |
| { | |
| "epoch": 0.47426924660354053, | |
| "grad_norm": 0.3982340395450592, | |
| "learning_rate": 3.137606921404191e-07, | |
| "logits/chosen": 10.228411674499512, | |
| "logits/rejected": 10.880895614624023, | |
| "logps/chosen": -85.34222412109375, | |
| "logps/ref_chosen": -107.1411361694336, | |
| "logps/ref_rejected": -118.66165161132812, | |
| "logps/rejected": -97.36593627929688, | |
| "loss": 0.4586, | |
| "rewards/accuracies": 0.515625, | |
| "rewards/chosen": 0.21798911690711975, | |
| "rewards/margins": 0.005031956359744072, | |
| "rewards/rejected": 0.21295715868473053, | |
| "sft_loss": 0.30310767889022827, | |
| "step": 36 | |
| }, | |
| { | |
| "epoch": 0.4874433923425278, | |
| "grad_norm": 0.24910680949687958, | |
| "learning_rate": 3.0236006569153616e-07, | |
| "logits/chosen": 10.425313949584961, | |
| "logits/rejected": 11.244641304016113, | |
| "logps/chosen": -85.0268325805664, | |
| "logps/ref_chosen": -106.6348876953125, | |
| "logps/ref_rejected": -121.37834167480469, | |
| "logps/rejected": -99.6484375, | |
| "loss": 0.4599, | |
| "rewards/accuracies": 0.546875, | |
| "rewards/chosen": 0.2160806506872177, | |
| "rewards/margins": -0.001218426157720387, | |
| "rewards/rejected": 0.21729910373687744, | |
| "sft_loss": 0.3029400110244751, | |
| "step": 37 | |
| }, | |
| { | |
| "epoch": 0.500617538081515, | |
| "grad_norm": 0.2774628698825836, | |
| "learning_rate": 2.9084434045463254e-07, | |
| "logits/chosen": 9.934067726135254, | |
| "logits/rejected": 11.023883819580078, | |
| "logps/chosen": -80.26193237304688, | |
| "logps/ref_chosen": -104.01033782958984, | |
| "logps/ref_rejected": -119.02666473388672, | |
| "logps/rejected": -94.75761413574219, | |
| "loss": 0.4611, | |
| "rewards/accuracies": 0.484375, | |
| "rewards/chosen": 0.23748423159122467, | |
| "rewards/margins": -0.005206258036196232, | |
| "rewards/rejected": 0.24269047379493713, | |
| "sft_loss": 0.30372247099876404, | |
| "step": 38 | |
| }, | |
| { | |
| "epoch": 0.5137916838205022, | |
| "grad_norm": 0.2735401391983032, | |
| "learning_rate": 2.7923883049302066e-07, | |
| "logits/chosen": 10.450285911560059, | |
| "logits/rejected": 11.239182472229004, | |
| "logps/chosen": -86.74555206298828, | |
| "logps/ref_chosen": -109.76485443115234, | |
| "logps/ref_rejected": -122.25163269042969, | |
| "logps/rejected": -100.94914245605469, | |
| "loss": 0.4595, | |
| "rewards/accuracies": 0.6171875, | |
| "rewards/chosen": 0.23019295930862427, | |
| "rewards/margins": 0.017168078571558, | |
| "rewards/rejected": 0.21302486956119537, | |
| "sft_loss": 0.3085777759552002, | |
| "step": 39 | |
| }, | |
| { | |
| "epoch": 0.5269658295594895, | |
| "grad_norm": 0.44081562757492065, | |
| "learning_rate": 2.6756904723632324e-07, | |
| "logits/chosen": 10.308364868164062, | |
| "logits/rejected": 11.480310440063477, | |
| "logps/chosen": -83.50869750976562, | |
| "logps/ref_chosen": -107.18782806396484, | |
| "logps/ref_rejected": -124.24542236328125, | |
| "logps/rejected": -101.8099594116211, | |
| "loss": 0.4537, | |
| "rewards/accuracies": 0.546875, | |
| "rewards/chosen": 0.23679129779338837, | |
| "rewards/margins": 0.012436658143997192, | |
| "rewards/rejected": 0.22435463964939117, | |
| "sft_loss": 0.29746735095977783, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.5401399752984768, | |
| "grad_norm": 0.2262677699327469, | |
| "learning_rate": 2.5586064340081516e-07, | |
| "logits/chosen": 10.629181861877441, | |
| "logits/rejected": 11.249412536621094, | |
| "logps/chosen": -83.5615463256836, | |
| "logps/ref_chosen": -106.42051696777344, | |
| "logps/ref_rejected": -122.25247192382812, | |
| "logps/rejected": -99.15153503417969, | |
| "loss": 0.4605, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": 0.22858966886997223, | |
| "rewards/margins": -0.0024197339080274105, | |
| "rewards/rejected": 0.23100940883159637, | |
| "sft_loss": 0.30392372608184814, | |
| "step": 41 | |
| }, | |
| { | |
| "epoch": 0.553314121037464, | |
| "grad_norm": 0.24259349703788757, | |
| "learning_rate": 2.4413935659918487e-07, | |
| "logits/chosen": 9.622812271118164, | |
| "logits/rejected": 10.690420150756836, | |
| "logps/chosen": -78.60126495361328, | |
| "logps/ref_chosen": -103.1148452758789, | |
| "logps/ref_rejected": -116.55464935302734, | |
| "logps/rejected": -93.2408676147461, | |
| "loss": 0.4435, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": 0.24513573944568634, | |
| "rewards/margins": 0.011997973546385765, | |
| "rewards/rejected": 0.23313775658607483, | |
| "sft_loss": 0.2801953852176666, | |
| "step": 42 | |
| }, | |
| { | |
| "epoch": 0.5664882667764513, | |
| "grad_norm": 0.259721577167511, | |
| "learning_rate": 2.3243095276367684e-07, | |
| "logits/chosen": 9.710105895996094, | |
| "logits/rejected": 10.863941192626953, | |
| "logps/chosen": -81.1871566772461, | |
| "logps/ref_chosen": -104.21064758300781, | |
| "logps/ref_rejected": -118.7614974975586, | |
| "logps/rejected": -95.96993255615234, | |
| "loss": 0.4518, | |
| "rewards/accuracies": 0.515625, | |
| "rewards/chosen": 0.23023495078086853, | |
| "rewards/margins": 0.0023193652741611004, | |
| "rewards/rejected": 0.22791558504104614, | |
| "sft_loss": 0.2908448278903961, | |
| "step": 43 | |
| }, | |
| { | |
| "epoch": 0.5796624125154385, | |
| "grad_norm": 0.20098231732845306, | |
| "learning_rate": 2.2076116950697937e-07, | |
| "logits/chosen": 9.823664665222168, | |
| "logits/rejected": 10.643632888793945, | |
| "logps/chosen": -76.53820037841797, | |
| "logps/ref_chosen": -100.59449005126953, | |
| "logps/ref_rejected": -115.95166778564453, | |
| "logps/rejected": -92.3114013671875, | |
| "loss": 0.4336, | |
| "rewards/accuracies": 0.5078125, | |
| "rewards/chosen": 0.24056285619735718, | |
| "rewards/margins": 0.004160105250775814, | |
| "rewards/rejected": 0.2364027351140976, | |
| "sft_loss": 0.26114290952682495, | |
| "step": 44 | |
| }, | |
| { | |
| "epoch": 0.5928365582544257, | |
| "grad_norm": 0.18732482194900513, | |
| "learning_rate": 2.091556595453674e-07, | |
| "logits/chosen": 10.034218788146973, | |
| "logits/rejected": 10.895869255065918, | |
| "logps/chosen": -82.88983917236328, | |
| "logps/ref_chosen": -106.96060943603516, | |
| "logps/ref_rejected": -125.49449157714844, | |
| "logps/rejected": -102.64103698730469, | |
| "loss": 0.4476, | |
| "rewards/accuracies": 0.6015625, | |
| "rewards/chosen": 0.24070771038532257, | |
| "rewards/margins": 0.012173159047961235, | |
| "rewards/rejected": 0.2285345494747162, | |
| "sft_loss": 0.28714537620544434, | |
| "step": 45 | |
| }, | |
| { | |
| "epoch": 0.606010703993413, | |
| "grad_norm": 0.21146298944950104, | |
| "learning_rate": 1.9763993430846392e-07, | |
| "logits/chosen": 10.091521263122559, | |
| "logits/rejected": 10.786705017089844, | |
| "logps/chosen": -81.19091796875, | |
| "logps/ref_chosen": -107.08544158935547, | |
| "logps/ref_rejected": -120.38542175292969, | |
| "logps/rejected": -94.10372924804688, | |
| "loss": 0.4533, | |
| "rewards/accuracies": 0.4453125, | |
| "rewards/chosen": 0.2589452862739563, | |
| "rewards/margins": -0.0038715943228453398, | |
| "rewards/rejected": 0.2628169059753418, | |
| "sft_loss": 0.291401743888855, | |
| "step": 46 | |
| }, | |
| { | |
| "epoch": 0.6191848497324002, | |
| "grad_norm": 0.1926412284374237, | |
| "learning_rate": 1.862393078595809e-07, | |
| "logits/chosen": 10.000544548034668, | |
| "logits/rejected": 11.2357177734375, | |
| "logps/chosen": -81.96806335449219, | |
| "logps/ref_chosen": -105.74787902832031, | |
| "logps/ref_rejected": -122.93606567382812, | |
| "logps/rejected": -100.6171875, | |
| "loss": 0.4428, | |
| "rewards/accuracies": 0.5546875, | |
| "rewards/chosen": 0.237798273563385, | |
| "rewards/margins": 0.01460958831012249, | |
| "rewards/rejected": 0.22318868339061737, | |
| "sft_loss": 0.2802280783653259, | |
| "step": 47 | |
| }, | |
| { | |
| "epoch": 0.6323589954713874, | |
| "grad_norm": 0.25539568066596985, | |
| "learning_rate": 1.7497884124991485e-07, | |
| "logits/chosen": 10.508065223693848, | |
| "logits/rejected": 11.458433151245117, | |
| "logps/chosen": -80.53589630126953, | |
| "logps/ref_chosen": -105.3005599975586, | |
| "logps/ref_rejected": -123.93569946289062, | |
| "logps/rejected": -98.65711212158203, | |
| "loss": 0.4509, | |
| "rewards/accuracies": 0.4921875, | |
| "rewards/chosen": 0.24764665961265564, | |
| "rewards/margins": -0.005139135755598545, | |
| "rewards/rejected": 0.2527858018875122, | |
| "sft_loss": 0.2870361804962158, | |
| "step": 48 | |
| }, | |
| { | |
| "epoch": 0.6455331412103746, | |
| "grad_norm": 0.2758500277996063, | |
| "learning_rate": 1.6388328742891678e-07, | |
| "logits/chosen": 10.53940486907959, | |
| "logits/rejected": 11.424980163574219, | |
| "logps/chosen": -78.68338775634766, | |
| "logps/ref_chosen": -104.30430603027344, | |
| "logps/ref_rejected": -115.85497283935547, | |
| "logps/rejected": -91.4198989868164, | |
| "loss": 0.4506, | |
| "rewards/accuracies": 0.4765625, | |
| "rewards/chosen": 0.25620919466018677, | |
| "rewards/margins": 0.011858467943966389, | |
| "rewards/rejected": 0.24435076117515564, | |
| "sft_loss": 0.2917044162750244, | |
| "step": 49 | |
| }, | |
| { | |
| "epoch": 0.6587072869493619, | |
| "grad_norm": 0.21196329593658447, | |
| "learning_rate": 1.5297703683193753e-07, | |
| "logits/chosen": 10.163137435913086, | |
| "logits/rejected": 11.043447494506836, | |
| "logps/chosen": -80.13683319091797, | |
| "logps/ref_chosen": -104.65946960449219, | |
| "logps/ref_rejected": -118.84170532226562, | |
| "logps/rejected": -94.65231323242188, | |
| "loss": 0.4413, | |
| "rewards/accuracies": 0.5078125, | |
| "rewards/chosen": 0.24522626399993896, | |
| "rewards/margins": 0.003332418156787753, | |
| "rewards/rejected": 0.24189382791519165, | |
| "sft_loss": 0.2736455202102661, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.6718814326883491, | |
| "grad_norm": 0.15545395016670227, | |
| "learning_rate": 1.422840637647574e-07, | |
| "logits/chosen": 10.250130653381348, | |
| "logits/rejected": 10.794774055480957, | |
| "logps/chosen": -79.24235534667969, | |
| "logps/ref_chosen": -104.4243392944336, | |
| "logps/ref_rejected": -117.16233825683594, | |
| "logps/rejected": -92.9332504272461, | |
| "loss": 0.4408, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": 0.25181978940963745, | |
| "rewards/margins": 0.009528912603855133, | |
| "rewards/rejected": 0.24229088425636292, | |
| "sft_loss": 0.27486634254455566, | |
| "step": 51 | |
| }, | |
| { | |
| "epoch": 0.6850555784273363, | |
| "grad_norm": 0.13252638280391693, | |
| "learning_rate": 1.3182787370285865e-07, | |
| "logits/chosen": 9.646485328674316, | |
| "logits/rejected": 10.813876152038574, | |
| "logps/chosen": -77.11428833007812, | |
| "logps/ref_chosen": -101.99165344238281, | |
| "logps/ref_rejected": -123.20516204833984, | |
| "logps/rejected": -98.14683532714844, | |
| "loss": 0.4409, | |
| "rewards/accuracies": 0.515625, | |
| "rewards/chosen": 0.24877360463142395, | |
| "rewards/margins": -0.001809664536267519, | |
| "rewards/rejected": 0.2505832612514496, | |
| "sft_loss": 0.2712918817996979, | |
| "step": 52 | |
| }, | |
| { | |
| "epoch": 0.6982297241663236, | |
| "grad_norm": 0.18613919615745544, | |
| "learning_rate": 1.2163145162128946e-07, | |
| "logits/chosen": 10.172059059143066, | |
| "logits/rejected": 11.077869415283203, | |
| "logps/chosen": -83.44391632080078, | |
| "logps/ref_chosen": -108.26175689697266, | |
| "logps/ref_rejected": -118.12374114990234, | |
| "logps/rejected": -93.78585052490234, | |
| "loss": 0.4409, | |
| "rewards/accuracies": 0.5234375, | |
| "rewards/chosen": 0.2481783926486969, | |
| "rewards/margins": 0.004799447022378445, | |
| "rewards/rejected": 0.24337893724441528, | |
| "sft_loss": 0.27333518862724304, | |
| "step": 53 | |
| }, | |
| { | |
| "epoch": 0.7114038699053108, | |
| "grad_norm": 0.2294066846370697, | |
| "learning_rate": 1.1171721146870014e-07, | |
| "logits/chosen": 10.27616024017334, | |
| "logits/rejected": 11.273432731628418, | |
| "logps/chosen": -81.40825653076172, | |
| "logps/ref_chosen": -108.5864028930664, | |
| "logps/ref_rejected": -130.25155639648438, | |
| "logps/rejected": -101.91203308105469, | |
| "loss": 0.4628, | |
| "rewards/accuracies": 0.3984375, | |
| "rewards/chosen": 0.27178147435188293, | |
| "rewards/margins": -0.011613852344453335, | |
| "rewards/rejected": 0.28339529037475586, | |
| "sft_loss": 0.3042542040348053, | |
| "step": 54 | |
| }, | |
| { | |
| "epoch": 0.724578015644298, | |
| "grad_norm": 0.1640205830335617, | |
| "learning_rate": 1.0210694689661939e-07, | |
| "logits/chosen": 10.290548324584961, | |
| "logits/rejected": 11.116561889648438, | |
| "logps/chosen": -79.66968536376953, | |
| "logps/ref_chosen": -105.69741821289062, | |
| "logps/ref_rejected": -122.07044219970703, | |
| "logps/rejected": -95.34159088134766, | |
| "loss": 0.4444, | |
| "rewards/accuracies": 0.4375, | |
| "rewards/chosen": 0.26027727127075195, | |
| "rewards/margins": -0.007011266425251961, | |
| "rewards/rejected": 0.26728853583335876, | |
| "sft_loss": 0.2752927243709564, | |
| "step": 55 | |
| }, | |
| { | |
| "epoch": 0.7377521613832853, | |
| "grad_norm": 0.2337108701467514, | |
| "learning_rate": 9.282178335227883e-08, | |
| "logits/chosen": 9.930572509765625, | |
| "logits/rejected": 11.117300033569336, | |
| "logps/chosen": -80.77919006347656, | |
| "logps/ref_chosen": -106.5007095336914, | |
| "logps/ref_rejected": -123.01736450195312, | |
| "logps/rejected": -97.2107162475586, | |
| "loss": 0.4534, | |
| "rewards/accuracies": 0.5390625, | |
| "rewards/chosen": 0.2572151720523834, | |
| "rewards/margins": -0.0008512809872627258, | |
| "rewards/rejected": 0.25806647539138794, | |
| "sft_loss": 0.29223573207855225, | |
| "step": 56 | |
| }, | |
| { | |
| "epoch": 0.7509263071222725, | |
| "grad_norm": 0.15076801180839539, | |
| "learning_rate": 8.388213164029459e-08, | |
| "logits/chosen": 10.707691192626953, | |
| "logits/rejected": 11.468770027160645, | |
| "logps/chosen": -84.24454498291016, | |
| "logps/ref_chosen": -109.18460083007812, | |
| "logps/ref_rejected": -124.3697280883789, | |
| "logps/rejected": -98.2645263671875, | |
| "loss": 0.4544, | |
| "rewards/accuracies": 0.46875, | |
| "rewards/chosen": 0.24940063059329987, | |
| "rewards/margins": -0.011651396751403809, | |
| "rewards/rejected": 0.26105204224586487, | |
| "sft_loss": 0.29044556617736816, | |
| "step": 57 | |
| }, | |
| { | |
| "epoch": 0.7641004528612598, | |
| "grad_norm": 0.3673655390739441, | |
| "learning_rate": 7.530764305528958e-08, | |
| "logits/chosen": 10.054794311523438, | |
| "logits/rejected": 10.671183586120605, | |
| "logps/chosen": -77.9069595336914, | |
| "logps/ref_chosen": -104.43944549560547, | |
| "logps/ref_rejected": -118.44985961914062, | |
| "logps/rejected": -90.52770233154297, | |
| "loss": 0.4475, | |
| "rewards/accuracies": 0.4609375, | |
| "rewards/chosen": 0.2653248608112335, | |
| "rewards/margins": -0.013896753080189228, | |
| "rewards/rejected": 0.2792215943336487, | |
| "sft_loss": 0.27804291248321533, | |
| "step": 58 | |
| }, | |
| { | |
| "epoch": 0.7772745986002471, | |
| "grad_norm": 0.21028165519237518, | |
| "learning_rate": 6.711716618408281e-08, | |
| "logits/chosen": 10.271055221557617, | |
| "logits/rejected": 11.227380752563477, | |
| "logps/chosen": -77.78079223632812, | |
| "logps/ref_chosen": -103.32658386230469, | |
| "logps/ref_rejected": -121.63726806640625, | |
| "logps/rejected": -95.64451599121094, | |
| "loss": 0.4511, | |
| "rewards/accuracies": 0.453125, | |
| "rewards/chosen": 0.2554578483104706, | |
| "rewards/margins": -0.004469675477594137, | |
| "rewards/rejected": 0.25992754101753235, | |
| "sft_loss": 0.2870379686355591, | |
| "step": 59 | |
| }, | |
| { | |
| "epoch": 0.7904487443392343, | |
| "grad_norm": 0.18925029039382935, | |
| "learning_rate": 5.932870547240454e-08, | |
| "logits/chosen": 10.035722732543945, | |
| "logits/rejected": 11.13114070892334, | |
| "logps/chosen": -76.91880798339844, | |
| "logps/ref_chosen": -102.98921966552734, | |
| "logps/ref_rejected": -124.47185516357422, | |
| "logps/rejected": -97.81532287597656, | |
| "loss": 0.4427, | |
| "rewards/accuracies": 0.5234375, | |
| "rewards/chosen": 0.26070404052734375, | |
| "rewards/margins": -0.005861295852810144, | |
| "rewards/rejected": 0.26656535267829895, | |
| "sft_loss": 0.27288156747817993, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.8036228900782215, | |
| "grad_norm": 0.2302037924528122, | |
| "learning_rate": 5.1959381647217665e-08, | |
| "logits/chosen": 10.10041618347168, | |
| "logits/rejected": 11.069572448730469, | |
| "logps/chosen": -78.97355651855469, | |
| "logps/ref_chosen": -106.28311157226562, | |
| "logps/ref_rejected": -121.47750854492188, | |
| "logps/rejected": -94.54800415039062, | |
| "loss": 0.4483, | |
| "rewards/accuracies": 0.515625, | |
| "rewards/chosen": 0.27309560775756836, | |
| "rewards/margins": 0.0038004759699106216, | |
| "rewards/rejected": 0.2692950963973999, | |
| "sft_loss": 0.2855130434036255, | |
| "step": 61 | |
| }, | |
| { | |
| "epoch": 0.8167970358172087, | |
| "grad_norm": 0.23445990681648254, | |
| "learning_rate": 4.502539408164385e-08, | |
| "logits/chosen": 10.193819046020508, | |
| "logits/rejected": 11.022415161132812, | |
| "logps/chosen": -82.26412200927734, | |
| "logps/ref_chosen": -109.67979431152344, | |
| "logps/ref_rejected": -120.36711120605469, | |
| "logps/rejected": -93.24017333984375, | |
| "loss": 0.4413, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.27415671944618225, | |
| "rewards/margins": 0.0028872936964035034, | |
| "rewards/rejected": 0.27126941084861755, | |
| "sft_loss": 0.27342402935028076, | |
| "step": 62 | |
| }, | |
| { | |
| "epoch": 0.829971181556196, | |
| "grad_norm": 0.6691642999649048, | |
| "learning_rate": 3.854198518522564e-08, | |
| "logits/chosen": 10.3654203414917, | |
| "logits/rejected": 11.456765174865723, | |
| "logps/chosen": -80.1414566040039, | |
| "logps/ref_chosen": -106.88896942138672, | |
| "logps/ref_rejected": -122.57796478271484, | |
| "logps/rejected": -97.27589416503906, | |
| "loss": 0.4494, | |
| "rewards/accuracies": 0.5390625, | |
| "rewards/chosen": 0.26747509837150574, | |
| "rewards/margins": 0.014454406686127186, | |
| "rewards/rejected": 0.25302067399024963, | |
| "sft_loss": 0.2905634641647339, | |
| "step": 63 | |
| }, | |
| { | |
| "epoch": 0.8431453272951832, | |
| "grad_norm": 0.22401364147663116, | |
| "learning_rate": 3.2523406897802444e-08, | |
| "logits/chosen": 10.32264232635498, | |
| "logits/rejected": 11.016410827636719, | |
| "logps/chosen": -83.06674194335938, | |
| "logps/ref_chosen": -109.29510498046875, | |
| "logps/ref_rejected": -121.51821899414062, | |
| "logps/rejected": -94.36360168457031, | |
| "loss": 0.455, | |
| "rewards/accuracies": 0.421875, | |
| "rewards/chosen": 0.26228344440460205, | |
| "rewards/margins": -0.00926260743290186, | |
| "rewards/rejected": 0.27154606580734253, | |
| "sft_loss": 0.2922515571117401, | |
| "step": 64 | |
| }, | |
| { | |
| "epoch": 0.8563194730341704, | |
| "grad_norm": 0.2678779661655426, | |
| "learning_rate": 2.6982889360653376e-08, | |
| "logits/chosen": 9.875053405761719, | |
| "logits/rejected": 11.186921119689941, | |
| "logps/chosen": -81.89527893066406, | |
| "logps/ref_chosen": -110.15232849121094, | |
| "logps/ref_rejected": -127.54279327392578, | |
| "logps/rejected": -101.82174682617188, | |
| "loss": 0.4476, | |
| "rewards/accuracies": 0.6015625, | |
| "rewards/chosen": 0.28257039189338684, | |
| "rewards/margins": 0.025359919294714928, | |
| "rewards/rejected": 0.25721046328544617, | |
| "sft_loss": 0.29137855768203735, | |
| "step": 65 | |
| }, | |
| { | |
| "epoch": 0.8694936187731577, | |
| "grad_norm": 0.2661888301372528, | |
| "learning_rate": 2.1932611833775843e-08, | |
| "logits/chosen": 9.89682388305664, | |
| "logits/rejected": 10.960628509521484, | |
| "logps/chosen": -77.75084686279297, | |
| "logps/ref_chosen": -104.0207748413086, | |
| "logps/ref_rejected": -126.93211364746094, | |
| "logps/rejected": -99.05423736572266, | |
| "loss": 0.4438, | |
| "rewards/accuracies": 0.46875, | |
| "rewards/chosen": 0.2626992464065552, | |
| "rewards/margins": -0.016079582273960114, | |
| "rewards/rejected": 0.2787788510322571, | |
| "sft_loss": 0.2711757719516754, | |
| "step": 66 | |
| }, | |
| { | |
| "epoch": 0.8826677645121449, | |
| "grad_norm": 0.19293785095214844, | |
| "learning_rate": 1.738367592322837e-08, | |
| "logits/chosen": 10.232080459594727, | |
| "logits/rejected": 11.185924530029297, | |
| "logps/chosen": -77.33721923828125, | |
| "logps/ref_chosen": -104.55751037597656, | |
| "logps/ref_rejected": -119.71514892578125, | |
| "logps/rejected": -93.18929290771484, | |
| "loss": 0.4416, | |
| "rewards/accuracies": 0.4921875, | |
| "rewards/chosen": 0.27220281958580017, | |
| "rewards/margins": 0.006944170221686363, | |
| "rewards/rejected": 0.26525866985321045, | |
| "sft_loss": 0.2752315402030945, | |
| "step": 67 | |
| }, | |
| { | |
| "epoch": 0.8958419102511321, | |
| "grad_norm": 0.24409052729606628, | |
| "learning_rate": 1.3346081177391472e-08, | |
| "logits/chosen": 10.561919212341309, | |
| "logits/rejected": 10.930088996887207, | |
| "logps/chosen": -81.20256042480469, | |
| "logps/ref_chosen": -107.26033020019531, | |
| "logps/ref_rejected": -115.8590087890625, | |
| "logps/rejected": -89.30443572998047, | |
| "loss": 0.4515, | |
| "rewards/accuracies": 0.46875, | |
| "rewards/chosen": 0.26057761907577515, | |
| "rewards/margins": -0.004968047607690096, | |
| "rewards/rejected": 0.26554566621780396, | |
| "sft_loss": 0.28793948888778687, | |
| "step": 68 | |
| }, | |
| { | |
| "epoch": 0.9090160559901194, | |
| "grad_norm": 0.3655894696712494, | |
| "learning_rate": 9.828703105789981e-09, | |
| "logits/chosen": 10.5272216796875, | |
| "logits/rejected": 11.433101654052734, | |
| "logps/chosen": -78.56044006347656, | |
| "logps/ref_chosen": -106.8610610961914, | |
| "logps/ref_rejected": -122.44428253173828, | |
| "logps/rejected": -95.06716918945312, | |
| "loss": 0.4496, | |
| "rewards/accuracies": 0.515625, | |
| "rewards/chosen": 0.28300610184669495, | |
| "rewards/margins": 0.00923481397330761, | |
| "rewards/rejected": 0.2737712860107422, | |
| "sft_loss": 0.2892051935195923, | |
| "step": 69 | |
| }, | |
| { | |
| "epoch": 0.9221902017291066, | |
| "grad_norm": 0.24153029918670654, | |
| "learning_rate": 6.839273668796747e-09, | |
| "logits/chosen": 10.204843521118164, | |
| "logits/rejected": 10.752978324890137, | |
| "logps/chosen": -81.29585266113281, | |
| "logps/ref_chosen": -106.58778381347656, | |
| "logps/ref_rejected": -124.77790832519531, | |
| "logps/rejected": -99.01206970214844, | |
| "loss": 0.446, | |
| "rewards/accuracies": 0.4921875, | |
| "rewards/chosen": 0.25291934609413147, | |
| "rewards/margins": -0.004739071242511272, | |
| "rewards/rejected": 0.2576584219932556, | |
| "sft_loss": 0.2786802649497986, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.9353643474680938, | |
| "grad_norm": 0.2623598277568817, | |
| "learning_rate": 4.384364281105973e-09, | |
| "logits/chosen": 10.224846839904785, | |
| "logits/rejected": 11.46517276763916, | |
| "logps/chosen": -76.17707824707031, | |
| "logps/ref_chosen": -104.39148712158203, | |
| "logps/ref_rejected": -120.59461212158203, | |
| "logps/rejected": -93.11611938476562, | |
| "loss": 0.4546, | |
| "rewards/accuracies": 0.5234375, | |
| "rewards/chosen": 0.28214412927627563, | |
| "rewards/margins": 0.007359158247709274, | |
| "rewards/rejected": 0.27478501200675964, | |
| "sft_loss": 0.29694974422454834, | |
| "step": 71 | |
| }, | |
| { | |
| "epoch": 0.9485384932070811, | |
| "grad_norm": 0.21674811840057373, | |
| "learning_rate": 2.469371366337264e-09, | |
| "logits/chosen": 10.320581436157227, | |
| "logits/rejected": 11.346898078918457, | |
| "logps/chosen": -82.47490692138672, | |
| "logps/ref_chosen": -108.53898620605469, | |
| "logps/ref_rejected": -123.26167297363281, | |
| "logps/rejected": -96.85331726074219, | |
| "loss": 0.4487, | |
| "rewards/accuracies": 0.46875, | |
| "rewards/chosen": 0.2606406807899475, | |
| "rewards/margins": -0.0034427556674927473, | |
| "rewards/rejected": 0.2640834450721741, | |
| "sft_loss": 0.28395283222198486, | |
| "step": 72 | |
| }, | |
| { | |
| "epoch": 0.9617126389460683, | |
| "grad_norm": 0.2906631827354431, | |
| "learning_rate": 1.0985044945254762e-09, | |
| "logits/chosen": 10.12803840637207, | |
| "logits/rejected": 10.97839641571045, | |
| "logps/chosen": -84.85684204101562, | |
| "logps/ref_chosen": -110.50595092773438, | |
| "logps/ref_rejected": -124.92510223388672, | |
| "logps/rejected": -99.70875549316406, | |
| "loss": 0.4422, | |
| "rewards/accuracies": 0.515625, | |
| "rewards/chosen": 0.256491094827652, | |
| "rewards/margins": 0.004327758681029081, | |
| "rewards/rejected": 0.2521633505821228, | |
| "sft_loss": 0.27563661336898804, | |
| "step": 73 | |
| }, | |
| { | |
| "epoch": 0.9748867846850556, | |
| "grad_norm": 0.33621564507484436, | |
| "learning_rate": 2.7477712857215675e-10, | |
| "logits/chosen": 10.371025085449219, | |
| "logits/rejected": 11.221382141113281, | |
| "logps/chosen": -80.57132720947266, | |
| "logps/ref_chosen": -107.1200942993164, | |
| "logps/ref_rejected": -120.22421264648438, | |
| "logps/rejected": -94.962158203125, | |
| "loss": 0.4398, | |
| "rewards/accuracies": 0.5390625, | |
| "rewards/chosen": 0.2654876708984375, | |
| "rewards/margins": 0.012867098674178123, | |
| "rewards/rejected": 0.2526205778121948, | |
| "sft_loss": 0.2743782699108124, | |
| "step": 74 | |
| }, | |
| { | |
| "epoch": 0.9880609304240429, | |
| "grad_norm": 0.1919887661933899, | |
| "learning_rate": 0.0, | |
| "logits/chosen": 10.14150333404541, | |
| "logits/rejected": 10.947701454162598, | |
| "logps/chosen": -80.30056762695312, | |
| "logps/ref_chosen": -107.40764617919922, | |
| "logps/ref_rejected": -120.6578369140625, | |
| "logps/rejected": -94.95838928222656, | |
| "loss": 0.4353, | |
| "rewards/accuracies": 0.5234375, | |
| "rewards/chosen": 0.2710708677768707, | |
| "rewards/margins": 0.01407632790505886, | |
| "rewards/rejected": 0.2569945454597473, | |
| "sft_loss": 0.2670779526233673, | |
| "step": 75 | |
| }, | |
| { | |
| "epoch": 0.9880609304240429, | |
| "step": 75, | |
| "total_flos": 0.0, | |
| "train_loss": 0.46543833255767825, | |
| "train_runtime": 8158.3981, | |
| "train_samples_per_second": 1.191, | |
| "train_steps_per_second": 0.009 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 75, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 12, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 2, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |