{ "best_metric": null, "best_model_checkpoint": null, "epoch": 0.47426924660354053, "eval_steps": 500, "global_step": 36, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "epoch": 0.013174145738987238, "grad_norm": 0.3937181234359741, "learning_rate": 6.25e-08, "logits/chosen": 10.038352012634277, "logits/rejected": 10.592904090881348, "logps/chosen": -132.56948852539062, "logps/ref_chosen": -132.56948852539062, "logps/ref_rejected": -148.778076171875, "logps/rejected": -148.778076171875, "loss": 0.6931, "rewards/accuracies": 0.0, "rewards/chosen": 0.0, "rewards/margins": 0.0, "rewards/rejected": 0.0, "step": 1 }, { "epoch": 0.026348291477974475, "grad_norm": 0.7554967999458313, "learning_rate": 1.25e-07, "logits/chosen": 10.320584297180176, "logits/rejected": 10.721946716308594, "logps/chosen": -132.46385192871094, "logps/ref_chosen": -132.46385192871094, "logps/ref_rejected": -142.69049072265625, "logps/rejected": -142.69049072265625, "loss": 0.6931, "rewards/accuracies": 0.0, "rewards/chosen": 0.0, "rewards/margins": 0.0, "rewards/rejected": 0.0, "step": 2 }, { "epoch": 0.03952243721696171, "grad_norm": 0.5059730410575867, "learning_rate": 1.875e-07, "logits/chosen": 9.891550064086914, "logits/rejected": 10.499095916748047, "logps/chosen": -136.52415466308594, "logps/ref_chosen": -135.79869079589844, "logps/ref_rejected": -145.69479370117188, "logps/rejected": -146.24124145507812, "loss": 0.6941, "rewards/accuracies": 0.3984375, "rewards/chosen": -0.007254539057612419, "rewards/margins": -0.0017902727704495192, "rewards/rejected": -0.005464266519993544, "step": 3 }, { "epoch": 0.05269658295594895, "grad_norm": 0.6780113577842712, "learning_rate": 2.5e-07, "logits/chosen": 10.122194290161133, "logits/rejected": 10.64792251586914, "logps/chosen": -137.41903686523438, "logps/ref_chosen": -136.85159301757812, "logps/ref_rejected": -142.64901733398438, "logps/rejected": -142.94912719726562, "loss": 0.6945, "rewards/accuracies": 0.46875, "rewards/chosen": -0.00567453121766448, "rewards/margins": -0.0026732482947409153, "rewards/rejected": -0.0030012831557542086, "step": 4 }, { "epoch": 0.06587072869493618, "grad_norm": 0.5103796124458313, "learning_rate": 3.1249999999999997e-07, "logits/chosen": 10.327558517456055, "logits/rejected": 11.079102516174316, "logps/chosen": -130.6144561767578, "logps/ref_chosen": -130.2961883544922, "logps/ref_rejected": -144.1498565673828, "logps/rejected": -144.40802001953125, "loss": 0.6935, "rewards/accuracies": 0.46875, "rewards/chosen": -0.0031829699873924255, "rewards/margins": -0.000601244973950088, "rewards/rejected": -0.0025817248970270157, "step": 5 }, { "epoch": 0.07904487443392343, "grad_norm": 0.7068921327590942, "learning_rate": 3.75e-07, "logits/chosen": 10.794892311096191, "logits/rejected": 11.326902389526367, "logps/chosen": -138.65377807617188, "logps/ref_chosen": -138.59861755371094, "logps/ref_rejected": -151.63775634765625, "logps/rejected": -151.48056030273438, "loss": 0.6942, "rewards/accuracies": 0.4765625, "rewards/chosen": -0.0005516067612916231, "rewards/margins": -0.002123636193573475, "rewards/rejected": 0.0015720291994512081, "step": 6 }, { "epoch": 0.09221902017291066, "grad_norm": 0.4639431834220886, "learning_rate": 4.375e-07, "logits/chosen": 9.949363708496094, "logits/rejected": 10.44249153137207, "logps/chosen": -130.56231689453125, "logps/ref_chosen": -129.5895538330078, "logps/ref_rejected": -139.67816162109375, "logps/rejected": -140.58245849609375, "loss": 0.6935, "rewards/accuracies": 0.5, "rewards/chosen": -0.009727789089083672, "rewards/margins": -0.000684772094246, "rewards/rejected": -0.009043017402291298, "step": 7 }, { "epoch": 0.1053931659118979, "grad_norm": 0.5582533478736877, "learning_rate": 5e-07, "logits/chosen": 10.655470848083496, "logits/rejected": 11.168412208557129, "logps/chosen": -134.7186737060547, "logps/ref_chosen": -134.24497985839844, "logps/ref_rejected": -147.19009399414062, "logps/rejected": -147.70596313476562, "loss": 0.693, "rewards/accuracies": 0.5078125, "rewards/chosen": -0.0047368621453642845, "rewards/margins": 0.0004215884255245328, "rewards/rejected": -0.0051584504544734955, "step": 8 }, { "epoch": 0.11856731165088513, "grad_norm": 0.559306263923645, "learning_rate": 4.997252228714278e-07, "logits/chosen": 10.525918006896973, "logits/rejected": 10.821675300598145, "logps/chosen": -142.31239318847656, "logps/ref_chosen": -141.59860229492188, "logps/ref_rejected": -149.33889770507812, "logps/rejected": -149.89822387695312, "loss": 0.6939, "rewards/accuracies": 0.4140625, "rewards/chosen": -0.007138015236705542, "rewards/margins": -0.001544784870930016, "rewards/rejected": -0.005593230482190847, "step": 9 }, { "epoch": 0.13174145738987236, "grad_norm": 0.6123551726341248, "learning_rate": 4.989014955054745e-07, "logits/chosen": 10.101277351379395, "logits/rejected": 10.55870246887207, "logps/chosen": -133.06532287597656, "logps/ref_chosen": -132.13449096679688, "logps/ref_rejected": -142.75973510742188, "logps/rejected": -143.90049743652344, "loss": 0.6921, "rewards/accuracies": 0.5390625, "rewards/chosen": -0.00930829904973507, "rewards/margins": 0.002099357545375824, "rewards/rejected": -0.011407657526433468, "step": 10 }, { "epoch": 0.14491560312885962, "grad_norm": 0.6729086637496948, "learning_rate": 4.975306286336627e-07, "logits/chosen": 10.170294761657715, "logits/rejected": 10.829442024230957, "logps/chosen": -137.3282012939453, "logps/ref_chosen": -135.85043334960938, "logps/ref_rejected": -152.14999389648438, "logps/rejected": -153.4501953125, "loss": 0.6941, "rewards/accuracies": 0.4375, "rewards/chosen": -0.014777721837162971, "rewards/margins": -0.0017757893074303865, "rewards/rejected": -0.013001934625208378, "step": 11 }, { "epoch": 0.15808974886784685, "grad_norm": 0.3486435115337372, "learning_rate": 4.956156357188939e-07, "logits/chosen": 10.261972427368164, "logits/rejected": 10.63994312286377, "logps/chosen": -141.65684509277344, "logps/ref_chosen": -139.63204956054688, "logps/ref_rejected": -148.03170776367188, "logps/rejected": -150.0493621826172, "loss": 0.6932, "rewards/accuracies": 0.484375, "rewards/chosen": -0.02024805173277855, "rewards/margins": -7.15737696737051e-05, "rewards/rejected": -0.0201764777302742, "step": 12 }, { "epoch": 0.17126389460683408, "grad_norm": 0.5037537813186646, "learning_rate": 4.931607263312032e-07, "logits/chosen": 9.811729431152344, "logits/rejected": 10.610340118408203, "logps/chosen": -131.98635864257812, "logps/ref_chosen": -129.56983947753906, "logps/ref_rejected": -143.01284790039062, "logps/rejected": -145.4197998046875, "loss": 0.6932, "rewards/accuracies": 0.5390625, "rewards/chosen": -0.024165067821741104, "rewards/margins": -9.549467358738184e-05, "rewards/rejected": -0.024069571867585182, "step": 13 }, { "epoch": 0.1844380403458213, "grad_norm": 1.0056744813919067, "learning_rate": 4.9017129689421e-07, "logits/chosen": 10.571147918701172, "logits/rejected": 11.290738105773926, "logps/chosen": -132.96786499023438, "logps/ref_chosen": -130.44677734375, "logps/ref_rejected": -142.0145721435547, "logps/rejected": -144.57803344726562, "loss": 0.693, "rewards/accuracies": 0.5, "rewards/chosen": -0.025210827589035034, "rewards/margins": 0.0004238423425704241, "rewards/rejected": -0.02563466690480709, "step": 14 }, { "epoch": 0.19761218608480857, "grad_norm": 0.44761258363723755, "learning_rate": 4.866539188226085e-07, "logits/chosen": 9.930328369140625, "logits/rejected": 10.632352828979492, "logps/chosen": -137.48495483398438, "logps/ref_chosen": -133.87620544433594, "logps/ref_rejected": -146.4284210205078, "logps/rejected": -150.177490234375, "loss": 0.6925, "rewards/accuracies": 0.5, "rewards/chosen": -0.036087535321712494, "rewards/margins": 0.0014031402533873916, "rewards/rejected": -0.03749067336320877, "step": 15 }, { "epoch": 0.2107863318237958, "grad_norm": 0.7480736970901489, "learning_rate": 4.826163240767716e-07, "logits/chosen": 10.873997688293457, "logits/rejected": 11.219457626342773, "logps/chosen": -138.66485595703125, "logps/ref_chosen": -135.57931518554688, "logps/ref_rejected": -144.543701171875, "logps/rejected": -147.69003295898438, "loss": 0.6929, "rewards/accuracies": 0.4765625, "rewards/chosen": -0.0308553334325552, "rewards/margins": 0.0006078272126615047, "rewards/rejected": -0.03146316111087799, "step": 16 }, { "epoch": 0.22396047756278303, "grad_norm": 0.5350177884101868, "learning_rate": 4.780673881662242e-07, "logits/chosen": 10.29094409942627, "logits/rejected": 10.750887870788574, "logps/chosen": -130.87939453125, "logps/ref_chosen": -127.53980255126953, "logps/ref_rejected": -141.44097900390625, "logps/rejected": -144.80685424804688, "loss": 0.6931, "rewards/accuracies": 0.5078125, "rewards/chosen": -0.03339584171772003, "rewards/margins": 0.0002629577647894621, "rewards/rejected": -0.03365879878401756, "step": 17 }, { "epoch": 0.23713462330177026, "grad_norm": 0.6819839477539062, "learning_rate": 4.730171106393466e-07, "logits/chosen": 10.341330528259277, "logits/rejected": 10.997025489807129, "logps/chosen": -135.26243591308594, "logps/ref_chosen": -130.62847900390625, "logps/ref_rejected": -141.88360595703125, "logps/rejected": -146.40476989746094, "loss": 0.6938, "rewards/accuracies": 0.484375, "rewards/chosen": -0.046339523047208786, "rewards/margins": -0.001128039206378162, "rewards/rejected": -0.045211486518383026, "step": 18 }, { "epoch": 0.2503087690407575, "grad_norm": 0.5013086199760437, "learning_rate": 4.6747659310219757e-07, "logits/chosen": 10.433052062988281, "logits/rejected": 10.780852317810059, "logps/chosen": -140.90821838378906, "logps/ref_chosen": -136.1767120361328, "logps/ref_rejected": -148.38580322265625, "logps/rejected": -153.79835510253906, "loss": 0.6898, "rewards/accuracies": 0.578125, "rewards/chosen": -0.0473150797188282, "rewards/margins": 0.006810260470956564, "rewards/rejected": -0.0541253462433815, "step": 19 }, { "epoch": 0.2634829147797447, "grad_norm": 0.5322529673576355, "learning_rate": 4.6145801481477433e-07, "logits/chosen": 10.486635208129883, "logits/rejected": 11.192235946655273, "logps/chosen": -140.9699249267578, "logps/ref_chosen": -134.8175048828125, "logps/ref_rejected": -145.92430114746094, "logps/rejected": -151.58935546875, "loss": 0.6957, "rewards/accuracies": 0.40625, "rewards/chosen": -0.06152408942580223, "rewards/margins": -0.004873568192124367, "rewards/rejected": -0.056650519371032715, "step": 20 }, { "epoch": 0.276657060518732, "grad_norm": 0.426982581615448, "learning_rate": 4.549746059183561e-07, "logits/chosen": 9.691137313842773, "logits/rejected": 10.519243240356445, "logps/chosen": -137.68182373046875, "logps/ref_chosen": -131.53555297851562, "logps/ref_rejected": -147.749267578125, "logps/rejected": -154.18560791015625, "loss": 0.6918, "rewards/accuracies": 0.5078125, "rewards/chosen": -0.061462655663490295, "rewards/margins": 0.002900664694607258, "rewards/rejected": -0.06436331570148468, "step": 21 }, { "epoch": 0.28983120625771924, "grad_norm": 0.39799365401268005, "learning_rate": 4.480406183527823e-07, "logits/chosen": 10.019172668457031, "logits/rejected": 10.592238426208496, "logps/chosen": -137.14170837402344, "logps/ref_chosen": -129.1536865234375, "logps/ref_rejected": -138.03895568847656, "logps/rejected": -145.4950714111328, "loss": 0.6959, "rewards/accuracies": 0.4609375, "rewards/chosen": -0.07987993955612183, "rewards/margins": -0.005318798590451479, "rewards/rejected": -0.07456114143133163, "step": 22 }, { "epoch": 0.3030053519967065, "grad_norm": 0.6805318593978882, "learning_rate": 4.4067129452759546e-07, "logits/chosen": 9.99167251586914, "logits/rejected": 10.793407440185547, "logps/chosen": -139.6283416748047, "logps/ref_chosen": -131.8021240234375, "logps/ref_rejected": -144.97862243652344, "logps/rejected": -152.63169860839844, "loss": 0.6942, "rewards/accuracies": 0.53125, "rewards/chosen": -0.07826196402311325, "rewards/margins": -0.0017311733681708574, "rewards/rejected": -0.07653079926967621, "step": 23 }, { "epoch": 0.3161794977356937, "grad_norm": 0.5804150104522705, "learning_rate": 4.3288283381591725e-07, "logits/chosen": 10.220123291015625, "logits/rejected": 10.670228958129883, "logps/chosen": -141.5391387939453, "logps/ref_chosen": -134.2378692626953, "logps/ref_rejected": -146.93955993652344, "logps/rejected": -154.45404052734375, "loss": 0.6922, "rewards/accuracies": 0.46875, "rewards/chosen": -0.07301273196935654, "rewards/margins": 0.0021319000516086817, "rewards/rejected": -0.07514464110136032, "step": 24 }, { "epoch": 0.32935364347468093, "grad_norm": 0.5388622283935547, "learning_rate": 4.246923569447104e-07, "logits/chosen": 10.13163948059082, "logits/rejected": 10.671570777893066, "logps/chosen": -143.1488037109375, "logps/ref_chosen": -134.34185791015625, "logps/ref_rejected": -152.0093231201172, "logps/rejected": -160.7644500732422, "loss": 0.6936, "rewards/accuracies": 0.4921875, "rewards/chosen": -0.08806941658258438, "rewards/margins": -0.0005181076703593135, "rewards/rejected": -0.0875513106584549, "step": 25 }, { "epoch": 0.34252778921366817, "grad_norm": 0.620239794254303, "learning_rate": 4.161178683597054e-07, "logits/chosen": 10.47970962524414, "logits/rejected": 11.029216766357422, "logps/chosen": -143.7899932861328, "logps/ref_chosen": -134.17701721191406, "logps/ref_rejected": -144.4202880859375, "logps/rejected": -153.90005493164062, "loss": 0.694, "rewards/accuracies": 0.4765625, "rewards/chosen": -0.09613002091646194, "rewards/margins": -0.0013323710300028324, "rewards/rejected": -0.09479765594005585, "step": 26 }, { "epoch": 0.3557019349526554, "grad_norm": 0.5931733250617981, "learning_rate": 4.0717821664772124e-07, "logits/chosen": 10.175558090209961, "logits/rejected": 10.997458457946777, "logps/chosen": -140.18865966796875, "logps/ref_chosen": -130.59686279296875, "logps/ref_rejected": -142.52255249023438, "logps/rejected": -152.0471954345703, "loss": 0.6937, "rewards/accuracies": 0.5234375, "rewards/chosen": -0.09591788053512573, "rewards/margins": -0.0006713568000122905, "rewards/rejected": -0.09524651616811752, "step": 27 }, { "epoch": 0.3688760806916426, "grad_norm": 0.5717474222183228, "learning_rate": 3.978930531033806e-07, "logits/chosen": 9.4828519821167, "logits/rejected": 10.328558921813965, "logps/chosen": -143.23362731933594, "logps/ref_chosen": -133.38320922851562, "logps/ref_rejected": -145.3930206298828, "logps/rejected": -155.37828063964844, "loss": 0.6927, "rewards/accuracies": 0.5390625, "rewards/chosen": -0.0985042154788971, "rewards/margins": 0.0013483850052580237, "rewards/rejected": -0.09985260665416718, "step": 28 }, { "epoch": 0.3820502264306299, "grad_norm": 0.4360070526599884, "learning_rate": 3.882827885312998e-07, "logits/chosen": 10.448491096496582, "logits/rejected": 10.812265396118164, "logps/chosen": -153.20803833007812, "logps/ref_chosen": -143.988037109375, "logps/ref_rejected": -152.1568603515625, "logps/rejected": -161.3251953125, "loss": 0.6936, "rewards/accuracies": 0.5, "rewards/chosen": -0.09220016002655029, "rewards/margins": -0.0005166500341147184, "rewards/rejected": -0.09168350696563721, "step": 29 }, { "epoch": 0.39522437216961714, "grad_norm": 0.4402547776699066, "learning_rate": 3.7836854837871044e-07, "logits/chosen": 10.051721572875977, "logits/rejected": 11.249682426452637, "logps/chosen": -142.13824462890625, "logps/ref_chosen": -132.1126251220703, "logps/ref_rejected": -152.75811767578125, "logps/rejected": -162.65602111816406, "loss": 0.694, "rewards/accuracies": 0.5078125, "rewards/chosen": -0.10025617480278015, "rewards/margins": -0.0012769806198775768, "rewards/rejected": -0.09897918999195099, "step": 30 }, { "epoch": 0.4083985179086044, "grad_norm": 0.48144519329071045, "learning_rate": 3.681721262971413e-07, "logits/chosen": 9.837646484375, "logits/rejected": 10.622282981872559, "logps/chosen": -138.56285095214844, "logps/ref_chosen": -128.07298278808594, "logps/ref_rejected": -143.27847290039062, "logps/rejected": -153.86830139160156, "loss": 0.6929, "rewards/accuracies": 0.5, "rewards/chosen": -0.10489849746227264, "rewards/margins": 0.000999803887680173, "rewards/rejected": -0.10589830577373505, "step": 31 }, { "epoch": 0.4215726636475916, "grad_norm": 0.577687680721283, "learning_rate": 3.577159362352426e-07, "logits/chosen": 9.869830131530762, "logits/rejected": 10.881388664245605, "logps/chosen": -138.81886291503906, "logps/ref_chosen": -128.7156982421875, "logps/ref_rejected": -145.83937072753906, "logps/rejected": -155.76939392089844, "loss": 0.6943, "rewards/accuracies": 0.4765625, "rewards/chosen": -0.10103171318769455, "rewards/margins": -0.0017315337900072336, "rewards/rejected": -0.09930018335580826, "step": 32 }, { "epoch": 0.43474680938657884, "grad_norm": 1.0172251462936401, "learning_rate": 3.470229631680624e-07, "logits/chosen": 9.960894584655762, "logits/rejected": 10.816852569580078, "logps/chosen": -147.22015380859375, "logps/ref_chosen": -135.9361572265625, "logps/ref_rejected": -147.2296600341797, "logps/rejected": -158.244873046875, "loss": 0.6947, "rewards/accuracies": 0.453125, "rewards/chosen": -0.1128399446606636, "rewards/margins": -0.002687737811356783, "rewards/rejected": -0.11015220731496811, "step": 33 }, { "epoch": 0.44792095512556607, "grad_norm": 0.6145241856575012, "learning_rate": 3.361167125710832e-07, "logits/chosen": 9.720129013061523, "logits/rejected": 10.441550254821777, "logps/chosen": -151.5580291748047, "logps/ref_chosen": -140.27676391601562, "logps/ref_rejected": -155.95626831054688, "logps/rejected": -167.70098876953125, "loss": 0.691, "rewards/accuracies": 0.546875, "rewards/chosen": -0.11281266808509827, "rewards/margins": 0.004634363576769829, "rewards/rejected": -0.11744703352451324, "step": 34 }, { "epoch": 0.4610951008645533, "grad_norm": 0.6173374652862549, "learning_rate": 3.2502115875008516e-07, "logits/chosen": 10.506845474243164, "logits/rejected": 10.996613502502441, "logps/chosen": -150.4586944580078, "logps/ref_chosen": -140.04530334472656, "logps/ref_rejected": -149.5904083251953, "logps/rejected": -160.94920349121094, "loss": 0.6887, "rewards/accuracies": 0.609375, "rewards/chosen": -0.10413391143083572, "rewards/margins": 0.009454021230340004, "rewards/rejected": -0.11358792334794998, "step": 35 }, { "epoch": 0.47426924660354053, "grad_norm": 0.5631862878799438, "learning_rate": 3.137606921404191e-07, "logits/chosen": 10.14232063293457, "logits/rejected": 10.531699180603027, "logps/chosen": -146.30441284179688, "logps/ref_chosen": -135.6486053466797, "logps/ref_rejected": -141.86065673828125, "logps/rejected": -151.99014282226562, "loss": 0.696, "rewards/accuracies": 0.5, "rewards/chosen": -0.10655791312456131, "rewards/margins": -0.005262960214167833, "rewards/rejected": -0.10129496455192566, "step": 36 } ], "logging_steps": 1, "max_steps": 75, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 12, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 2, "trial_name": null, "trial_params": null }