| { | |
| "best_metric": 0.9497246742248535, | |
| "best_model_checkpoint": "saves/Vicuna-7B-v1.5/lora/orpo-salt/checkpoint-1500", | |
| "epoch": 2.9969690846635686, | |
| "eval_steps": 500, | |
| "global_step": 1854, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.01616488179430188, | |
| "grad_norm": 0.3899887204170227, | |
| "learning_rate": 4.999648198770648e-06, | |
| "logits/chosen": -0.8260404467582703, | |
| "logits/rejected": -0.779380202293396, | |
| "logps/chosen": -1.0734994411468506, | |
| "logps/rejected": -1.2254035472869873, | |
| "loss": 1.146, | |
| "odds_ratio_loss": 0.7249619364738464, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.1073499470949173, | |
| "rewards/margins": 0.01519041694700718, | |
| "rewards/rejected": -0.12254035472869873, | |
| "sft_loss": 1.0734994411468506, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.03232976358860376, | |
| "grad_norm": 0.4923989176750183, | |
| "learning_rate": 4.998578646361359e-06, | |
| "logits/chosen": -0.7854002714157104, | |
| "logits/rejected": -0.781389594078064, | |
| "logps/chosen": -1.0866433382034302, | |
| "logps/rejected": -1.2551138401031494, | |
| "loss": 1.1535, | |
| "odds_ratio_loss": 0.668422520160675, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.10866433382034302, | |
| "rewards/margins": 0.016847047954797745, | |
| "rewards/rejected": -0.12551137804985046, | |
| "sft_loss": 1.0866433382034302, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.04849464538290564, | |
| "grad_norm": 0.7084988951683044, | |
| "learning_rate": 4.996791614004449e-06, | |
| "logits/chosen": -0.7559419274330139, | |
| "logits/rejected": -0.7485054731369019, | |
| "logps/chosen": -1.0929394960403442, | |
| "logps/rejected": -1.1501963138580322, | |
| "loss": 1.1699, | |
| "odds_ratio_loss": 0.7694913148880005, | |
| "rewards/accuracies": 0.42500001192092896, | |
| "rewards/chosen": -0.10929396003484726, | |
| "rewards/margins": 0.00572569016367197, | |
| "rewards/rejected": -0.11501964181661606, | |
| "sft_loss": 1.0929394960403442, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.06465952717720752, | |
| "grad_norm": 0.8286219239234924, | |
| "learning_rate": 4.994287614855618e-06, | |
| "logits/chosen": -0.8193706274032593, | |
| "logits/rejected": -0.7897969484329224, | |
| "logps/chosen": -1.1362740993499756, | |
| "logps/rejected": -1.1394835710525513, | |
| "loss": 1.2171, | |
| "odds_ratio_loss": 0.808376133441925, | |
| "rewards/accuracies": 0.48124998807907104, | |
| "rewards/chosen": -0.11362739652395248, | |
| "rewards/margins": 0.0003209514543414116, | |
| "rewards/rejected": -0.11394836008548737, | |
| "sft_loss": 1.1362740993499756, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.0808244089715094, | |
| "grad_norm": 0.537628173828125, | |
| "learning_rate": 4.991067367951343e-06, | |
| "logits/chosen": -0.7530331015586853, | |
| "logits/rejected": -0.7703112363815308, | |
| "logps/chosen": -1.0968067646026611, | |
| "logps/rejected": -1.1828521490097046, | |
| "loss": 1.1729, | |
| "odds_ratio_loss": 0.7610759735107422, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.10968067497015, | |
| "rewards/margins": 0.00860452838242054, | |
| "rewards/rejected": -0.11828521639108658, | |
| "sft_loss": 1.0968067646026611, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.09698929076581128, | |
| "grad_norm": 0.2992643415927887, | |
| "learning_rate": 4.987131798002389e-06, | |
| "logits/chosen": -0.7554941773414612, | |
| "logits/rejected": -0.7805821299552917, | |
| "logps/chosen": -1.120224118232727, | |
| "logps/rejected": -1.1958564519882202, | |
| "loss": 1.2007, | |
| "odds_ratio_loss": 0.804762065410614, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.11202241480350494, | |
| "rewards/margins": 0.007563246879726648, | |
| "rewards/rejected": -0.11958565562963486, | |
| "sft_loss": 1.120224118232727, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.11315417256011315, | |
| "grad_norm": 0.5207487940788269, | |
| "learning_rate": 4.982482035128285e-06, | |
| "logits/chosen": -0.7931987643241882, | |
| "logits/rejected": -0.7725004553794861, | |
| "logps/chosen": -1.158760666847229, | |
| "logps/rejected": -1.3085857629776, | |
| "loss": 1.2342, | |
| "odds_ratio_loss": 0.7545939683914185, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.11587607860565186, | |
| "rewards/margins": 0.01498250663280487, | |
| "rewards/rejected": -0.13085858523845673, | |
| "sft_loss": 1.158760666847229, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.12931905435441504, | |
| "grad_norm": 0.8179022669792175, | |
| "learning_rate": 4.9771194145328e-06, | |
| "logits/chosen": -0.7553219199180603, | |
| "logits/rejected": -0.7355794906616211, | |
| "logps/chosen": -0.9810718297958374, | |
| "logps/rejected": -1.1142699718475342, | |
| "loss": 1.0496, | |
| "odds_ratio_loss": 0.6851751208305359, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.09810719639062881, | |
| "rewards/margins": 0.013319805264472961, | |
| "rewards/rejected": -0.11142698675394058, | |
| "sft_loss": 0.9810718297958374, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.1454839361487169, | |
| "grad_norm": 0.5893221497535706, | |
| "learning_rate": 4.971045476120532e-06, | |
| "logits/chosen": -0.7767540216445923, | |
| "logits/rejected": -0.7691196203231812, | |
| "logps/chosen": -1.0343536138534546, | |
| "logps/rejected": -1.1126210689544678, | |
| "loss": 1.1086, | |
| "odds_ratio_loss": 0.7424803972244263, | |
| "rewards/accuracies": 0.4937500059604645, | |
| "rewards/chosen": -0.10343535989522934, | |
| "rewards/margins": 0.007826738059520721, | |
| "rewards/rejected": -0.11126209795475006, | |
| "sft_loss": 1.0343536138534546, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.1616488179430188, | |
| "grad_norm": 0.3746645748615265, | |
| "learning_rate": 4.964261964054713e-06, | |
| "logits/chosen": -0.749561607837677, | |
| "logits/rejected": -0.7426966428756714, | |
| "logps/chosen": -1.0808948278427124, | |
| "logps/rejected": -1.1608020067214966, | |
| "loss": 1.1637, | |
| "odds_ratio_loss": 0.8280612826347351, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.10808948427438736, | |
| "rewards/margins": 0.007990716025233269, | |
| "rewards/rejected": -0.11608020961284637, | |
| "sft_loss": 1.0808948278427124, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.17781369973732067, | |
| "grad_norm": 0.5266828536987305, | |
| "learning_rate": 4.956770826256372e-06, | |
| "logits/chosen": -0.7276872396469116, | |
| "logits/rejected": -0.7239276766777039, | |
| "logps/chosen": -1.0891507863998413, | |
| "logps/rejected": -1.188951015472412, | |
| "loss": 1.1606, | |
| "odds_ratio_loss": 0.7148129940032959, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.10891509056091309, | |
| "rewards/margins": 0.009980013594031334, | |
| "rewards/rejected": -0.11889511346817017, | |
| "sft_loss": 1.0891507863998413, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.19397858153162256, | |
| "grad_norm": 0.5117731690406799, | |
| "learning_rate": 4.94857421384497e-06, | |
| "logits/chosen": -0.7153638601303101, | |
| "logits/rejected": -0.7017214894294739, | |
| "logps/chosen": -1.0659247636795044, | |
| "logps/rejected": -1.1995283365249634, | |
| "loss": 1.1411, | |
| "odds_ratio_loss": 0.7518999576568604, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.10659247636795044, | |
| "rewards/margins": 0.013360358774662018, | |
| "rewards/rejected": -0.11995282024145126, | |
| "sft_loss": 1.0659247636795044, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.21014346332592443, | |
| "grad_norm": 0.3964090049266815, | |
| "learning_rate": 4.939674480520701e-06, | |
| "logits/chosen": -0.7281032800674438, | |
| "logits/rejected": -0.6757130026817322, | |
| "logps/chosen": -0.9924377202987671, | |
| "logps/rejected": -1.0807675123214722, | |
| "loss": 1.0644, | |
| "odds_ratio_loss": 0.7199574708938599, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.09924378246068954, | |
| "rewards/margins": 0.008832980878651142, | |
| "rewards/rejected": -0.10807675123214722, | |
| "sft_loss": 0.9924377202987671, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.2263083451202263, | |
| "grad_norm": 0.31593117117881775, | |
| "learning_rate": 4.930074181888613e-06, | |
| "logits/chosen": -0.6932573914527893, | |
| "logits/rejected": -0.6765223741531372, | |
| "logps/chosen": -1.011648416519165, | |
| "logps/rejected": -1.1101162433624268, | |
| "loss": 1.0811, | |
| "odds_ratio_loss": 0.6949580907821655, | |
| "rewards/accuracies": 0.4937500059604645, | |
| "rewards/chosen": -0.10116484016180038, | |
| "rewards/margins": 0.009846789762377739, | |
| "rewards/rejected": -0.11101162433624268, | |
| "sft_loss": 1.011648416519165, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.2424732269145282, | |
| "grad_norm": 0.7396884560585022, | |
| "learning_rate": 4.91977607472475e-06, | |
| "logits/chosen": -0.6414996981620789, | |
| "logits/rejected": -0.6007689237594604, | |
| "logps/chosen": -1.0180175304412842, | |
| "logps/rejected": -1.0574676990509033, | |
| "loss": 1.0929, | |
| "odds_ratio_loss": 0.748645544052124, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.10180176794528961, | |
| "rewards/margins": 0.00394500233232975, | |
| "rewards/rejected": -0.10574676841497421, | |
| "sft_loss": 1.0180175304412842, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.2586381087088301, | |
| "grad_norm": 0.5049052834510803, | |
| "learning_rate": 4.908783116184534e-06, | |
| "logits/chosen": -0.6661972403526306, | |
| "logits/rejected": -0.626873791217804, | |
| "logps/chosen": -0.953465461730957, | |
| "logps/rejected": -1.0835082530975342, | |
| "loss": 1.02, | |
| "odds_ratio_loss": 0.6655644178390503, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.09534655511379242, | |
| "rewards/margins": 0.013004262931644917, | |
| "rewards/rejected": -0.10835081338882446, | |
| "sft_loss": 0.953465461730957, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 0.27480299050313195, | |
| "grad_norm": 0.4969651699066162, | |
| "learning_rate": 4.897098462953598e-06, | |
| "logits/chosen": -0.5929690599441528, | |
| "logits/rejected": -0.6147447824478149, | |
| "logps/chosen": -0.9747630953788757, | |
| "logps/rejected": -1.1718312501907349, | |
| "loss": 1.0464, | |
| "odds_ratio_loss": 0.7164822220802307, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.09747631102800369, | |
| "rewards/margins": 0.019706813618540764, | |
| "rewards/rejected": -0.11718311160802841, | |
| "sft_loss": 0.9747630953788757, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 0.2909678722974338, | |
| "grad_norm": 0.37429389357566833, | |
| "learning_rate": 4.884725470341331e-06, | |
| "logits/chosen": -0.5573834180831909, | |
| "logits/rejected": -0.544479250907898, | |
| "logps/chosen": -0.8867887258529663, | |
| "logps/rejected": -1.1076356172561646, | |
| "loss": 0.9499, | |
| "odds_ratio_loss": 0.6307954788208008, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.08867888152599335, | |
| "rewards/margins": 0.022084690630435944, | |
| "rewards/rejected": -0.1107635647058487, | |
| "sft_loss": 0.8867887258529663, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 0.3071327540917357, | |
| "grad_norm": 1.2109434604644775, | |
| "learning_rate": 4.871667691317377e-06, | |
| "logits/chosen": -0.6222495436668396, | |
| "logits/rejected": -0.6174622774124146, | |
| "logps/chosen": -1.1702499389648438, | |
| "logps/rejected": -1.0528119802474976, | |
| "loss": 1.2649, | |
| "odds_ratio_loss": 0.9465614557266235, | |
| "rewards/accuracies": 0.40625, | |
| "rewards/chosen": -0.1170249953866005, | |
| "rewards/margins": -0.011743778362870216, | |
| "rewards/rejected": -0.10528121143579483, | |
| "sft_loss": 1.1702499389648438, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 0.3232976358860376, | |
| "grad_norm": 1.5371562242507935, | |
| "learning_rate": 4.857928875491392e-06, | |
| "logits/chosen": -0.5464112162590027, | |
| "logits/rejected": -0.5513696670532227, | |
| "logps/chosen": -0.8908155560493469, | |
| "logps/rejected": -1.0076180696487427, | |
| "loss": 0.9612, | |
| "odds_ratio_loss": 0.7040323615074158, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.08908155560493469, | |
| "rewards/margins": 0.011680259369313717, | |
| "rewards/rejected": -0.10076181590557098, | |
| "sft_loss": 0.8908155560493469, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.33946251768033947, | |
| "grad_norm": 0.6159927845001221, | |
| "learning_rate": 4.843512968036314e-06, | |
| "logits/chosen": -0.6329461932182312, | |
| "logits/rejected": -0.592659592628479, | |
| "logps/chosen": -0.975503146648407, | |
| "logps/rejected": -0.9970613718032837, | |
| "loss": 1.0514, | |
| "odds_ratio_loss": 0.7591590881347656, | |
| "rewards/accuracies": 0.4937500059604645, | |
| "rewards/chosen": -0.09755031019449234, | |
| "rewards/margins": 0.00215582805685699, | |
| "rewards/rejected": -0.09970613569021225, | |
| "sft_loss": 0.975503146648407, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 0.35562739947464134, | |
| "grad_norm": 0.3111410439014435, | |
| "learning_rate": 4.828424108555486e-06, | |
| "logits/chosen": -0.5221891403198242, | |
| "logits/rejected": -0.5304391980171204, | |
| "logps/chosen": -1.1862733364105225, | |
| "logps/rejected": -1.2753493785858154, | |
| "loss": 1.2641, | |
| "odds_ratio_loss": 0.7783994674682617, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.11862732470035553, | |
| "rewards/margins": 0.008907611481845379, | |
| "rewards/rejected": -0.12753494083881378, | |
| "sft_loss": 1.1862733364105225, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 0.3717922812689432, | |
| "grad_norm": 0.301698237657547, | |
| "learning_rate": 4.812666629893957e-06, | |
| "logits/chosen": -0.4992770254611969, | |
| "logits/rejected": -0.4967115521430969, | |
| "logps/chosen": -0.9971933364868164, | |
| "logps/rejected": -1.0213407278060913, | |
| "loss": 1.0744, | |
| "odds_ratio_loss": 0.7721298933029175, | |
| "rewards/accuracies": 0.44999998807907104, | |
| "rewards/chosen": -0.09971933811903, | |
| "rewards/margins": 0.0024147380609065294, | |
| "rewards/rejected": -0.1021340861916542, | |
| "sft_loss": 0.9971933364868164, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 0.3879571630632451, | |
| "grad_norm": 0.2967057526111603, | |
| "learning_rate": 4.796245056894273e-06, | |
| "logits/chosen": -0.5198571085929871, | |
| "logits/rejected": -0.4987764358520508, | |
| "logps/chosen": -0.9578666687011719, | |
| "logps/rejected": -1.0644018650054932, | |
| "loss": 1.0315, | |
| "odds_ratio_loss": 0.7367077469825745, | |
| "rewards/accuracies": 0.4749999940395355, | |
| "rewards/chosen": -0.0957866758108139, | |
| "rewards/margins": 0.01065351627767086, | |
| "rewards/rejected": -0.10644018650054932, | |
| "sft_loss": 0.9578666687011719, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 0.404122044857547, | |
| "grad_norm": 0.336041659116745, | |
| "learning_rate": 4.779164105097148e-06, | |
| "logits/chosen": -0.4748106002807617, | |
| "logits/rejected": -0.44636374711990356, | |
| "logps/chosen": -0.9247462153434753, | |
| "logps/rejected": -1.1018692255020142, | |
| "loss": 0.9923, | |
| "odds_ratio_loss": 0.6758453845977783, | |
| "rewards/accuracies": 0.4937500059604645, | |
| "rewards/chosen": -0.09247462451457977, | |
| "rewards/margins": 0.017712296918034554, | |
| "rewards/rejected": -0.11018691956996918, | |
| "sft_loss": 0.9247462153434753, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 0.42028692665184886, | |
| "grad_norm": 0.5222122669219971, | |
| "learning_rate": 4.761428679387373e-06, | |
| "logits/chosen": -0.46434497833251953, | |
| "logits/rejected": -0.4350043833255768, | |
| "logps/chosen": -0.8905488848686218, | |
| "logps/rejected": -1.0182609558105469, | |
| "loss": 0.9591, | |
| "odds_ratio_loss": 0.6853379011154175, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.0890548899769783, | |
| "rewards/margins": 0.01277120690792799, | |
| "rewards/rejected": -0.10182609409093857, | |
| "sft_loss": 0.8905488848686218, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 0.4364518084461507, | |
| "grad_norm": 0.5936411023139954, | |
| "learning_rate": 4.7430438725853515e-06, | |
| "logits/chosen": -0.48627519607543945, | |
| "logits/rejected": -0.4379982352256775, | |
| "logps/chosen": -0.9183929562568665, | |
| "logps/rejected": -1.1679961681365967, | |
| "loss": 0.984, | |
| "odds_ratio_loss": 0.6556900143623352, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.09183929860591888, | |
| "rewards/margins": 0.024960322305560112, | |
| "rewards/rejected": -0.11679961532354355, | |
| "sft_loss": 0.9183929562568665, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 0.4526166902404526, | |
| "grad_norm": 0.46239179372787476, | |
| "learning_rate": 4.724014963984669e-06, | |
| "logits/chosen": -0.4012899398803711, | |
| "logits/rejected": -0.411139577627182, | |
| "logps/chosen": -1.008721947669983, | |
| "logps/rejected": -1.2014849185943604, | |
| "loss": 1.0765, | |
| "odds_ratio_loss": 0.6780184507369995, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.10087219625711441, | |
| "rewards/margins": 0.01927630603313446, | |
| "rewards/rejected": -0.12014850229024887, | |
| "sft_loss": 1.008721947669983, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 0.4687815720347545, | |
| "grad_norm": 0.5760877132415771, | |
| "learning_rate": 4.704347417836116e-06, | |
| "logits/chosen": -0.4533885419368744, | |
| "logits/rejected": -0.46080097556114197, | |
| "logps/chosen": -0.9372620582580566, | |
| "logps/rejected": -1.1106752157211304, | |
| "loss": 1.0089, | |
| "odds_ratio_loss": 0.716440737247467, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.09372620284557343, | |
| "rewards/margins": 0.01734132692217827, | |
| "rewards/rejected": -0.1110675185918808, | |
| "sft_loss": 0.9372620582580566, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 0.4849464538290564, | |
| "grad_norm": 0.44260743260383606, | |
| "learning_rate": 4.684046881778603e-06, | |
| "logits/chosen": -0.5344091653823853, | |
| "logits/rejected": -0.49474531412124634, | |
| "logps/chosen": -0.9150590896606445, | |
| "logps/rejected": -1.0017120838165283, | |
| "loss": 0.9833, | |
| "odds_ratio_loss": 0.6827279329299927, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.09150592237710953, | |
| "rewards/margins": 0.008665294386446476, | |
| "rewards/rejected": -0.10017120838165283, | |
| "sft_loss": 0.9150590896606445, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.5011113356233583, | |
| "grad_norm": 0.3225099742412567, | |
| "learning_rate": 4.663119185217409e-06, | |
| "logits/chosen": -0.43460625410079956, | |
| "logits/rejected": -0.4127863049507141, | |
| "logps/chosen": -0.8891846537590027, | |
| "logps/rejected": -1.0905497074127197, | |
| "loss": 0.954, | |
| "odds_ratio_loss": 0.6476849913597107, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.08891846239566803, | |
| "rewards/margins": 0.020136509090662003, | |
| "rewards/rejected": -0.10905496776103973, | |
| "sft_loss": 0.8891846537590027, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 0.5172762174176602, | |
| "grad_norm": 0.3512892723083496, | |
| "learning_rate": 4.641570337650232e-06, | |
| "logits/chosen": -0.43388432264328003, | |
| "logits/rejected": -0.39495667815208435, | |
| "logps/chosen": -0.8790934681892395, | |
| "logps/rejected": -0.9963566064834595, | |
| "loss": 0.9498, | |
| "odds_ratio_loss": 0.7069565057754517, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.08790934085845947, | |
| "rewards/margins": 0.011726310476660728, | |
| "rewards/rejected": -0.09963564574718475, | |
| "sft_loss": 0.8790934681892395, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 0.533441099211962, | |
| "grad_norm": 0.3520517349243164, | |
| "learning_rate": 4.61940652694154e-06, | |
| "logits/chosen": -0.45831650495529175, | |
| "logits/rejected": -0.4600452780723572, | |
| "logps/chosen": -0.9612126350402832, | |
| "logps/rejected": -1.0601940155029297, | |
| "loss": 1.0373, | |
| "odds_ratio_loss": 0.7606214880943298, | |
| "rewards/accuracies": 0.4749999940395355, | |
| "rewards/chosen": -0.09612125903367996, | |
| "rewards/margins": 0.00989813357591629, | |
| "rewards/rejected": -0.10601940006017685, | |
| "sft_loss": 0.9612126350402832, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 0.5496059810062639, | |
| "grad_norm": 0.42445889115333557, | |
| "learning_rate": 4.596634117545689e-06, | |
| "logits/chosen": -0.3920242190361023, | |
| "logits/rejected": -0.41387075185775757, | |
| "logps/chosen": -0.9238036274909973, | |
| "logps/rejected": -1.0761339664459229, | |
| "loss": 0.9917, | |
| "odds_ratio_loss": 0.6789978742599487, | |
| "rewards/accuracies": 0.4749999940395355, | |
| "rewards/chosen": -0.09238035976886749, | |
| "rewards/margins": 0.015233027748763561, | |
| "rewards/rejected": -0.10761336982250214, | |
| "sft_loss": 0.9238036274909973, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 0.5657708628005658, | |
| "grad_norm": 0.3377890884876251, | |
| "learning_rate": 4.573259648679335e-06, | |
| "logits/chosen": -0.39150765538215637, | |
| "logits/rejected": -0.4451742172241211, | |
| "logps/chosen": -0.9269700050354004, | |
| "logps/rejected": -1.077823281288147, | |
| "loss": 0.9987, | |
| "odds_ratio_loss": 0.7173791527748108, | |
| "rewards/accuracies": 0.48124998807907104, | |
| "rewards/chosen": -0.09269699454307556, | |
| "rewards/margins": 0.01508533675223589, | |
| "rewards/rejected": -0.10778234153985977, | |
| "sft_loss": 0.9269700050354004, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 0.5819357445948676, | |
| "grad_norm": 0.9352906942367554, | |
| "learning_rate": 4.549289832443663e-06, | |
| "logits/chosen": -0.39780086278915405, | |
| "logits/rejected": -0.3602847754955292, | |
| "logps/chosen": -0.9020577669143677, | |
| "logps/rejected": -1.0630056858062744, | |
| "loss": 0.9737, | |
| "odds_ratio_loss": 0.7168340682983398, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.09020576626062393, | |
| "rewards/margins": 0.01609480008482933, | |
| "rewards/rejected": -0.10630057752132416, | |
| "sft_loss": 0.9020577669143677, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 0.5981006263891695, | |
| "grad_norm": 0.3642963469028473, | |
| "learning_rate": 4.524731551896978e-06, | |
| "logits/chosen": -0.4040652811527252, | |
| "logits/rejected": -0.39201897382736206, | |
| "logps/chosen": -0.822562038898468, | |
| "logps/rejected": -0.9484196901321411, | |
| "loss": 0.8918, | |
| "odds_ratio_loss": 0.6919523477554321, | |
| "rewards/accuracies": 0.4937500059604645, | |
| "rewards/chosen": -0.08225620537996292, | |
| "rewards/margins": 0.012585763819515705, | |
| "rewards/rejected": -0.09484197199344635, | |
| "sft_loss": 0.822562038898468, | |
| "step": 370 | |
| }, | |
| { | |
| "epoch": 0.6142655081834714, | |
| "grad_norm": 0.9358541965484619, | |
| "learning_rate": 4.4995918590781925e-06, | |
| "logits/chosen": -0.41558751463890076, | |
| "logits/rejected": -0.39345669746398926, | |
| "logps/chosen": -0.9379288554191589, | |
| "logps/rejected": -1.0011296272277832, | |
| "loss": 1.0132, | |
| "odds_ratio_loss": 0.7530064582824707, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.0937928855419159, | |
| "rewards/margins": 0.006320066750049591, | |
| "rewards/rejected": -0.10011295974254608, | |
| "sft_loss": 0.9379288554191589, | |
| "step": 380 | |
| }, | |
| { | |
| "epoch": 0.6304303899777733, | |
| "grad_norm": 0.42754364013671875, | |
| "learning_rate": 4.473877972981797e-06, | |
| "logits/chosen": -0.4294399321079254, | |
| "logits/rejected": -0.48693591356277466, | |
| "logps/chosen": -0.9050455093383789, | |
| "logps/rejected": -1.0990797281265259, | |
| "loss": 0.9681, | |
| "odds_ratio_loss": 0.6305026412010193, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.09050454199314117, | |
| "rewards/margins": 0.019403431564569473, | |
| "rewards/rejected": -0.10990796983242035, | |
| "sft_loss": 0.9050455093383789, | |
| "step": 390 | |
| }, | |
| { | |
| "epoch": 0.6465952717720752, | |
| "grad_norm": 0.3870018422603607, | |
| "learning_rate": 4.447597277484894e-06, | |
| "logits/chosen": -0.41894254088401794, | |
| "logits/rejected": -0.3863012492656708, | |
| "logps/chosen": -0.9011236429214478, | |
| "logps/rejected": -1.011643648147583, | |
| "loss": 0.971, | |
| "odds_ratio_loss": 0.6992276906967163, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.0901123657822609, | |
| "rewards/margins": 0.011052015237510204, | |
| "rewards/rejected": -0.10116437822580338, | |
| "sft_loss": 0.9011236429214478, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 0.6627601535663771, | |
| "grad_norm": 0.6716357469558716, | |
| "learning_rate": 4.42075731922687e-06, | |
| "logits/chosen": -0.381665974855423, | |
| "logits/rejected": -0.40627461671829224, | |
| "logps/chosen": -0.9860145449638367, | |
| "logps/rejected": -1.0734965801239014, | |
| "loss": 1.0559, | |
| "odds_ratio_loss": 0.6987608671188354, | |
| "rewards/accuracies": 0.4749999940395355, | |
| "rewards/chosen": -0.09860144555568695, | |
| "rewards/margins": 0.008748206309974194, | |
| "rewards/rejected": -0.10734964907169342, | |
| "sft_loss": 0.9860145449638367, | |
| "step": 410 | |
| }, | |
| { | |
| "epoch": 0.6789250353606789, | |
| "grad_norm": 0.4379284083843231, | |
| "learning_rate": 4.3933658054423465e-06, | |
| "logits/chosen": -0.42450767755508423, | |
| "logits/rejected": -0.4302968978881836, | |
| "logps/chosen": -0.8682054281234741, | |
| "logps/rejected": -1.0158107280731201, | |
| "loss": 0.9348, | |
| "odds_ratio_loss": 0.6656124591827393, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.0868205577135086, | |
| "rewards/margins": 0.014760518446564674, | |
| "rewards/rejected": -0.10158105939626694, | |
| "sft_loss": 0.8682054281234741, | |
| "step": 420 | |
| }, | |
| { | |
| "epoch": 0.6950899171549808, | |
| "grad_norm": 0.4341568052768707, | |
| "learning_rate": 4.365430601748003e-06, | |
| "logits/chosen": -0.3941816985607147, | |
| "logits/rejected": -0.349882036447525, | |
| "logps/chosen": -0.9646803140640259, | |
| "logps/rejected": -1.0113680362701416, | |
| "loss": 1.0372, | |
| "odds_ratio_loss": 0.7253597974777222, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.09646803140640259, | |
| "rewards/margins": 0.004668788518756628, | |
| "rewards/rejected": -0.10113681852817535, | |
| "sft_loss": 0.9646803140640259, | |
| "step": 430 | |
| }, | |
| { | |
| "epoch": 0.7112547989492827, | |
| "grad_norm": 1.7109006643295288, | |
| "learning_rate": 4.336959729883925e-06, | |
| "logits/chosen": -0.37049371004104614, | |
| "logits/rejected": -0.3737342953681946, | |
| "logps/chosen": -0.9116461873054504, | |
| "logps/rejected": -0.9422439336776733, | |
| "loss": 0.9849, | |
| "odds_ratio_loss": 0.7329493165016174, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.09116461873054504, | |
| "rewards/margins": 0.0030597783625125885, | |
| "rewards/rejected": -0.09422439336776733, | |
| "sft_loss": 0.9116461873054504, | |
| "step": 440 | |
| }, | |
| { | |
| "epoch": 0.7274196807435845, | |
| "grad_norm": 0.4295767843723297, | |
| "learning_rate": 4.307961365410118e-06, | |
| "logits/chosen": -0.46054011583328247, | |
| "logits/rejected": -0.4506424069404602, | |
| "logps/chosen": -0.904135525226593, | |
| "logps/rejected": -0.965890109539032, | |
| "loss": 0.9756, | |
| "odds_ratio_loss": 0.7150284051895142, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.09041355550289154, | |
| "rewards/margins": 0.0061754509806632996, | |
| "rewards/rejected": -0.09658900648355484, | |
| "sft_loss": 0.904135525226593, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 0.7435845625378864, | |
| "grad_norm": 0.7776443958282471, | |
| "learning_rate": 4.278443835358854e-06, | |
| "logits/chosen": -0.3951818645000458, | |
| "logits/rejected": -0.4040835499763489, | |
| "logps/chosen": -0.8823555707931519, | |
| "logps/rejected": -1.1062017679214478, | |
| "loss": 0.9449, | |
| "odds_ratio_loss": 0.6257806420326233, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.08823557198047638, | |
| "rewards/margins": 0.022384602576494217, | |
| "rewards/rejected": -0.1106201782822609, | |
| "sft_loss": 0.8823555707931519, | |
| "step": 460 | |
| }, | |
| { | |
| "epoch": 0.7597494443321883, | |
| "grad_norm": 0.37953025102615356, | |
| "learning_rate": 4.248415615843523e-06, | |
| "logits/chosen": -0.376980721950531, | |
| "logits/rejected": -0.40178006887435913, | |
| "logps/chosen": -0.9119707345962524, | |
| "logps/rejected": -0.9874213933944702, | |
| "loss": 0.9817, | |
| "odds_ratio_loss": 0.6976627111434937, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.09119707345962524, | |
| "rewards/margins": 0.007545073516666889, | |
| "rewards/rejected": -0.09874214231967926, | |
| "sft_loss": 0.9119707345962524, | |
| "step": 470 | |
| }, | |
| { | |
| "epoch": 0.7759143261264903, | |
| "grad_norm": 0.5314805507659912, | |
| "learning_rate": 4.217885329624666e-06, | |
| "logits/chosen": -0.3499462604522705, | |
| "logits/rejected": -0.33436357975006104, | |
| "logps/chosen": -0.876055121421814, | |
| "logps/rejected": -1.064893126487732, | |
| "loss": 0.9413, | |
| "odds_ratio_loss": 0.6526578068733215, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.08760551363229752, | |
| "rewards/margins": 0.018883811309933662, | |
| "rewards/rejected": -0.10648931562900543, | |
| "sft_loss": 0.876055121421814, | |
| "step": 480 | |
| }, | |
| { | |
| "epoch": 0.7920792079207921, | |
| "grad_norm": 0.40282461047172546, | |
| "learning_rate": 4.186861743633911e-06, | |
| "logits/chosen": -0.41591471433639526, | |
| "logits/rejected": -0.4058813154697418, | |
| "logps/chosen": -0.8972100019454956, | |
| "logps/rejected": -1.093335509300232, | |
| "loss": 0.9699, | |
| "odds_ratio_loss": 0.7265552282333374, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.08972100913524628, | |
| "rewards/margins": 0.01961255446076393, | |
| "rewards/rejected": -0.10933355987071991, | |
| "sft_loss": 0.8972100019454956, | |
| "step": 490 | |
| }, | |
| { | |
| "epoch": 0.808244089715094, | |
| "grad_norm": 0.43431738018989563, | |
| "learning_rate": 4.155353766456497e-06, | |
| "logits/chosen": -0.30508697032928467, | |
| "logits/rejected": -0.3136020302772522, | |
| "logps/chosen": -0.9303945302963257, | |
| "logps/rejected": -1.0141643285751343, | |
| "loss": 1.0008, | |
| "odds_ratio_loss": 0.7037394046783447, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.09303945302963257, | |
| "rewards/margins": 0.00837697833776474, | |
| "rewards/rejected": -0.1014164462685585, | |
| "sft_loss": 0.9303945302963257, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 0.808244089715094, | |
| "eval_logits/chosen": -0.3878052830696106, | |
| "eval_logits/rejected": -0.3689490258693695, | |
| "eval_logps/chosen": -0.9066087007522583, | |
| "eval_logps/rejected": -1.0192701816558838, | |
| "eval_loss": 0.9776538014411926, | |
| "eval_odds_ratio_loss": 0.710451602935791, | |
| "eval_rewards/accuracies": 0.5054545402526855, | |
| "eval_rewards/chosen": -0.09066087007522583, | |
| "eval_rewards/margins": 0.011266152374446392, | |
| "eval_rewards/rejected": -0.1019270196557045, | |
| "eval_runtime": 192.2826, | |
| "eval_samples_per_second": 5.721, | |
| "eval_sft_loss": 0.9066087007522583, | |
| "eval_steps_per_second": 2.86, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 0.8244089715093958, | |
| "grad_norm": 0.3983856737613678, | |
| "learning_rate": 4.123370445773134e-06, | |
| "logits/chosen": -0.344710111618042, | |
| "logits/rejected": -0.3169902563095093, | |
| "logps/chosen": -0.8998648524284363, | |
| "logps/rejected": -0.9106130599975586, | |
| "loss": 0.975, | |
| "odds_ratio_loss": 0.7513402700424194, | |
| "rewards/accuracies": 0.48750001192092896, | |
| "rewards/chosen": -0.08998648822307587, | |
| "rewards/margins": 0.0010748239001259208, | |
| "rewards/rejected": -0.0910613164305687, | |
| "sft_loss": 0.8998648524284363, | |
| "step": 510 | |
| }, | |
| { | |
| "epoch": 0.8405738533036977, | |
| "grad_norm": 1.2135205268859863, | |
| "learning_rate": 4.090920965761906e-06, | |
| "logits/chosen": -0.3448580205440521, | |
| "logits/rejected": -0.3548375964164734, | |
| "logps/chosen": -0.9812738299369812, | |
| "logps/rejected": -1.0694336891174316, | |
| "loss": 1.0535, | |
| "odds_ratio_loss": 0.7224698662757874, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.09812740236520767, | |
| "rewards/margins": 0.00881598424166441, | |
| "rewards/rejected": -0.10694338381290436, | |
| "sft_loss": 0.9812738299369812, | |
| "step": 520 | |
| }, | |
| { | |
| "epoch": 0.8567387350979996, | |
| "grad_norm": 0.9130859375, | |
| "learning_rate": 4.058014644460991e-06, | |
| "logits/chosen": -0.34060588479042053, | |
| "logits/rejected": -0.3562433123588562, | |
| "logps/chosen": -0.9648042917251587, | |
| "logps/rejected": -1.0603010654449463, | |
| "loss": 1.032, | |
| "odds_ratio_loss": 0.6720489859580994, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.09648042917251587, | |
| "rewards/margins": 0.009549676440656185, | |
| "rewards/rejected": -0.10603010654449463, | |
| "sft_loss": 0.9648042917251587, | |
| "step": 530 | |
| }, | |
| { | |
| "epoch": 0.8729036168923014, | |
| "grad_norm": 0.6945879459381104, | |
| "learning_rate": 4.024660931092939e-06, | |
| "logits/chosen": -0.39998704195022583, | |
| "logits/rejected": -0.39360350370407104, | |
| "logps/chosen": -0.8902137875556946, | |
| "logps/rejected": -1.0513432025909424, | |
| "loss": 0.9562, | |
| "odds_ratio_loss": 0.6595617532730103, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.08902137726545334, | |
| "rewards/margins": 0.016112947836518288, | |
| "rewards/rejected": -0.10513432323932648, | |
| "sft_loss": 0.8902137875556946, | |
| "step": 540 | |
| }, | |
| { | |
| "epoch": 0.8890684986866033, | |
| "grad_norm": 0.45378220081329346, | |
| "learning_rate": 3.990869403351272e-06, | |
| "logits/chosen": -0.3531869053840637, | |
| "logits/rejected": -0.38131508231163025, | |
| "logps/chosen": -0.9068384170532227, | |
| "logps/rejected": -1.065394639968872, | |
| "loss": 0.9704, | |
| "odds_ratio_loss": 0.635545015335083, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.09068384021520615, | |
| "rewards/margins": 0.01585562154650688, | |
| "rewards/rejected": -0.10653946548700333, | |
| "sft_loss": 0.9068384170532227, | |
| "step": 550 | |
| }, | |
| { | |
| "epoch": 0.9052333804809052, | |
| "grad_norm": 0.5512678623199463, | |
| "learning_rate": 3.956649764650206e-06, | |
| "logits/chosen": -0.29515427350997925, | |
| "logits/rejected": -0.31435275077819824, | |
| "logps/chosen": -0.9203943014144897, | |
| "logps/rejected": -1.0603986978530884, | |
| "loss": 0.9918, | |
| "odds_ratio_loss": 0.7142159938812256, | |
| "rewards/accuracies": 0.46875, | |
| "rewards/chosen": -0.09203943610191345, | |
| "rewards/margins": 0.014000418595969677, | |
| "rewards/rejected": -0.106039859354496, | |
| "sft_loss": 0.9203943014144897, | |
| "step": 560 | |
| }, | |
| { | |
| "epoch": 0.9213982622752072, | |
| "grad_norm": 0.5750080347061157, | |
| "learning_rate": 3.92201184133826e-06, | |
| "logits/chosen": -0.3182484209537506, | |
| "logits/rejected": -0.3164721131324768, | |
| "logps/chosen": -0.8570343255996704, | |
| "logps/rejected": -1.0225125551223755, | |
| "loss": 0.922, | |
| "odds_ratio_loss": 0.6495530009269714, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.08570344746112823, | |
| "rewards/margins": 0.01654782146215439, | |
| "rewards/rejected": -0.10225125402212143, | |
| "sft_loss": 0.8570343255996704, | |
| "step": 570 | |
| }, | |
| { | |
| "epoch": 0.937563144069509, | |
| "grad_norm": 0.5823240876197815, | |
| "learning_rate": 3.886965579876572e-06, | |
| "logits/chosen": -0.307335764169693, | |
| "logits/rejected": -0.331511914730072, | |
| "logps/chosen": -0.8535898923873901, | |
| "logps/rejected": -0.9173160791397095, | |
| "loss": 0.9234, | |
| "odds_ratio_loss": 0.6983198523521423, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.08535899966955185, | |
| "rewards/margins": 0.006372606847435236, | |
| "rewards/rejected": -0.09173160046339035, | |
| "sft_loss": 0.8535898923873901, | |
| "step": 580 | |
| }, | |
| { | |
| "epoch": 0.9537280258638109, | |
| "grad_norm": 0.3793308734893799, | |
| "learning_rate": 3.851521043982716e-06, | |
| "logits/chosen": -0.3546546399593353, | |
| "logits/rejected": -0.3105318248271942, | |
| "logps/chosen": -0.9257644414901733, | |
| "logps/rejected": -0.994279682636261, | |
| "loss": 0.9977, | |
| "odds_ratio_loss": 0.7192004919052124, | |
| "rewards/accuracies": 0.46875, | |
| "rewards/chosen": -0.09257644414901733, | |
| "rewards/margins": 0.006851526442915201, | |
| "rewards/rejected": -0.0994279757142067, | |
| "sft_loss": 0.9257644414901733, | |
| "step": 590 | |
| }, | |
| { | |
| "epoch": 0.9698929076581128, | |
| "grad_norm": 0.5789406895637512, | |
| "learning_rate": 3.81568841174086e-06, | |
| "logits/chosen": -0.39430108666419983, | |
| "logits/rejected": -0.38088011741638184, | |
| "logps/chosen": -0.8874362111091614, | |
| "logps/rejected": -1.0097267627716064, | |
| "loss": 0.9592, | |
| "odds_ratio_loss": 0.7179639935493469, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.08874362707138062, | |
| "rewards/margins": 0.012229054234921932, | |
| "rewards/rejected": -0.10097268968820572, | |
| "sft_loss": 0.8874362111091614, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 0.9860577894524146, | |
| "grad_norm": 0.4505593478679657, | |
| "learning_rate": 3.7794779726790664e-06, | |
| "logits/chosen": -0.4197085499763489, | |
| "logits/rejected": -0.3544057607650757, | |
| "logps/chosen": -0.8556501269340515, | |
| "logps/rejected": -0.9688836336135864, | |
| "loss": 0.9233, | |
| "odds_ratio_loss": 0.6760933995246887, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.0855650082230568, | |
| "rewards/margins": 0.011323352344334126, | |
| "rewards/rejected": -0.09688836336135864, | |
| "sft_loss": 0.8556501269340515, | |
| "step": 610 | |
| }, | |
| { | |
| "epoch": 1.0022226712467166, | |
| "grad_norm": 0.41950830817222595, | |
| "learning_rate": 3.7429001248146096e-06, | |
| "logits/chosen": -0.3560163080692291, | |
| "logits/rejected": -0.32193905115127563, | |
| "logps/chosen": -0.8660818934440613, | |
| "logps/rejected": -1.0638062953948975, | |
| "loss": 0.9302, | |
| "odds_ratio_loss": 0.6412297487258911, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.08660819381475449, | |
| "rewards/margins": 0.019772443920373917, | |
| "rewards/rejected": -0.10638062655925751, | |
| "sft_loss": 0.8660818934440613, | |
| "step": 620 | |
| }, | |
| { | |
| "epoch": 1.0183875530410185, | |
| "grad_norm": 0.30259978771209717, | |
| "learning_rate": 3.7059653716681227e-06, | |
| "logits/chosen": -0.3218996524810791, | |
| "logits/rejected": -0.3514016568660736, | |
| "logps/chosen": -0.9751222729682922, | |
| "logps/rejected": -1.1278547048568726, | |
| "loss": 1.046, | |
| "odds_ratio_loss": 0.7084661722183228, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.09751223772764206, | |
| "rewards/margins": 0.015273240394890308, | |
| "rewards/rejected": -0.11278548091650009, | |
| "sft_loss": 0.9751222729682922, | |
| "step": 630 | |
| }, | |
| { | |
| "epoch": 1.0345524348353203, | |
| "grad_norm": 1.449523687362671, | |
| "learning_rate": 3.668684319247463e-06, | |
| "logits/chosen": -0.3402321934700012, | |
| "logits/rejected": -0.3320569396018982, | |
| "logps/chosen": -0.8782706260681152, | |
| "logps/rejected": -1.0504738092422485, | |
| "loss": 0.9434, | |
| "odds_ratio_loss": 0.651136040687561, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.08782706409692764, | |
| "rewards/margins": 0.01722031459212303, | |
| "rewards/rejected": -0.10504738241434097, | |
| "sft_loss": 0.8782706260681152, | |
| "step": 640 | |
| }, | |
| { | |
| "epoch": 1.0507173166296222, | |
| "grad_norm": 0.36652296781539917, | |
| "learning_rate": 3.6310676730021373e-06, | |
| "logits/chosen": -0.3392433524131775, | |
| "logits/rejected": -0.3268556296825409, | |
| "logps/chosen": -0.8789156079292297, | |
| "logps/rejected": -0.9153023958206177, | |
| "loss": 0.9515, | |
| "odds_ratio_loss": 0.7262720465660095, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.08789155632257462, | |
| "rewards/margins": 0.0036386798601597548, | |
| "rewards/rejected": -0.09153024852275848, | |
| "sft_loss": 0.8789156079292297, | |
| "step": 650 | |
| }, | |
| { | |
| "epoch": 1.066882198423924, | |
| "grad_norm": 0.42644253373146057, | |
| "learning_rate": 3.593126234749178e-06, | |
| "logits/chosen": -0.35958123207092285, | |
| "logits/rejected": -0.33439984917640686, | |
| "logps/chosen": -0.9317266345024109, | |
| "logps/rejected": -0.9812437891960144, | |
| "loss": 1.004, | |
| "odds_ratio_loss": 0.7226861119270325, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.09317266196012497, | |
| "rewards/margins": 0.0049517154693603516, | |
| "rewards/rejected": -0.09812436997890472, | |
| "sft_loss": 0.9317266345024109, | |
| "step": 660 | |
| }, | |
| { | |
| "epoch": 1.083047080218226, | |
| "grad_norm": 0.5300435423851013, | |
| "learning_rate": 3.554870899571343e-06, | |
| "logits/chosen": -0.4070967137813568, | |
| "logits/rejected": -0.38338038325309753, | |
| "logps/chosen": -0.9088705778121948, | |
| "logps/rejected": -1.0065948963165283, | |
| "loss": 0.9774, | |
| "odds_ratio_loss": 0.6850352883338928, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.09088706225156784, | |
| "rewards/margins": 0.009772435761988163, | |
| "rewards/rejected": -0.10065948963165283, | |
| "sft_loss": 0.9088705778121948, | |
| "step": 670 | |
| }, | |
| { | |
| "epoch": 1.0992119620125278, | |
| "grad_norm": 1.5718979835510254, | |
| "learning_rate": 3.5163126526885373e-06, | |
| "logits/chosen": -0.3708317279815674, | |
| "logits/rejected": -0.3510357737541199, | |
| "logps/chosen": -0.8702448606491089, | |
| "logps/rejected": -0.9972399473190308, | |
| "loss": 0.9409, | |
| "odds_ratio_loss": 0.7065256834030151, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.08702448755502701, | |
| "rewards/margins": 0.012699509970843792, | |
| "rewards/rejected": -0.09972399473190308, | |
| "sft_loss": 0.8702448606491089, | |
| "step": 680 | |
| }, | |
| { | |
| "epoch": 1.1153768438068297, | |
| "grad_norm": 0.31913694739341736, | |
| "learning_rate": 3.4774625663033484e-06, | |
| "logits/chosen": -0.39085036516189575, | |
| "logits/rejected": -0.37611085176467896, | |
| "logps/chosen": -0.8731836080551147, | |
| "logps/rejected": -0.9660570025444031, | |
| "loss": 0.9427, | |
| "odds_ratio_loss": 0.6954530477523804, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.08731836825609207, | |
| "rewards/margins": 0.009287341497838497, | |
| "rewards/rejected": -0.09660570323467255, | |
| "sft_loss": 0.8731836080551147, | |
| "step": 690 | |
| }, | |
| { | |
| "epoch": 1.1315417256011315, | |
| "grad_norm": 0.5645192265510559, | |
| "learning_rate": 3.4383317964216067e-06, | |
| "logits/chosen": -0.3893832564353943, | |
| "logits/rejected": -0.3442583680152893, | |
| "logps/chosen": -0.870397686958313, | |
| "logps/rejected": -0.9214354753494263, | |
| "loss": 0.9448, | |
| "odds_ratio_loss": 0.7436445355415344, | |
| "rewards/accuracies": 0.44999998807907104, | |
| "rewards/chosen": -0.0870397686958313, | |
| "rewards/margins": 0.005103783216327429, | |
| "rewards/rejected": -0.09214354306459427, | |
| "sft_loss": 0.870397686958313, | |
| "step": 700 | |
| }, | |
| { | |
| "epoch": 1.1477066073954334, | |
| "grad_norm": 0.7822654247283936, | |
| "learning_rate": 3.398931579648877e-06, | |
| "logits/chosen": -0.3577522039413452, | |
| "logits/rejected": -0.2890363931655884, | |
| "logps/chosen": -0.9082385301589966, | |
| "logps/rejected": -1.1010273694992065, | |
| "loss": 0.9792, | |
| "odds_ratio_loss": 0.7092560529708862, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.09082385897636414, | |
| "rewards/margins": 0.019278880208730698, | |
| "rewards/rejected": -0.11010273545980453, | |
| "sft_loss": 0.9082385301589966, | |
| "step": 710 | |
| }, | |
| { | |
| "epoch": 1.1638714891897353, | |
| "grad_norm": 0.6916553974151611, | |
| "learning_rate": 3.359273229963813e-06, | |
| "logits/chosen": -0.33050891757011414, | |
| "logits/rejected": -0.33249133825302124, | |
| "logps/chosen": -0.8524163961410522, | |
| "logps/rejected": -0.9603297114372253, | |
| "loss": 0.9215, | |
| "odds_ratio_loss": 0.6913267374038696, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.0852416455745697, | |
| "rewards/margins": 0.010791336186230183, | |
| "rewards/rejected": -0.09603297710418701, | |
| "sft_loss": 0.8524163961410522, | |
| "step": 720 | |
| }, | |
| { | |
| "epoch": 1.1800363709840371, | |
| "grad_norm": 0.36541640758514404, | |
| "learning_rate": 3.319368135469285e-06, | |
| "logits/chosen": -0.34484899044036865, | |
| "logits/rejected": -0.3120992183685303, | |
| "logps/chosen": -0.8964350819587708, | |
| "logps/rejected": -1.0409529209136963, | |
| "loss": 0.9665, | |
| "odds_ratio_loss": 0.7009326219558716, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.08964350074529648, | |
| "rewards/margins": 0.014451777562499046, | |
| "rewards/rejected": -0.10409528017044067, | |
| "sft_loss": 0.8964350819587708, | |
| "step": 730 | |
| }, | |
| { | |
| "epoch": 1.196201252778339, | |
| "grad_norm": 0.5928468704223633, | |
| "learning_rate": 3.279227755122228e-06, | |
| "logits/chosen": -0.359285831451416, | |
| "logits/rejected": -0.3708931505680084, | |
| "logps/chosen": -0.817459225654602, | |
| "logps/rejected": -1.1048064231872559, | |
| "loss": 0.8791, | |
| "odds_ratio_loss": 0.6168545484542847, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.0817459225654602, | |
| "rewards/margins": 0.028734717518091202, | |
| "rewards/rejected": -0.1104806438088417, | |
| "sft_loss": 0.817459225654602, | |
| "step": 740 | |
| }, | |
| { | |
| "epoch": 1.2123661345726409, | |
| "grad_norm": 0.8944354057312012, | |
| "learning_rate": 3.2388636154431417e-06, | |
| "logits/chosen": -0.32971471548080444, | |
| "logits/rejected": -0.3240662217140198, | |
| "logps/chosen": -0.9531005024909973, | |
| "logps/rejected": -1.1055543422698975, | |
| "loss": 1.0252, | |
| "odds_ratio_loss": 0.7207925319671631, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.09531004726886749, | |
| "rewards/margins": 0.015245395712554455, | |
| "rewards/rejected": -0.11055544763803482, | |
| "sft_loss": 0.9531005024909973, | |
| "step": 750 | |
| }, | |
| { | |
| "epoch": 1.2285310163669427, | |
| "grad_norm": 0.5451232194900513, | |
| "learning_rate": 3.198287307206192e-06, | |
| "logits/chosen": -0.3906642198562622, | |
| "logits/rejected": -0.36378178000450134, | |
| "logps/chosen": -0.909538745880127, | |
| "logps/rejected": -1.005489706993103, | |
| "loss": 0.9791, | |
| "odds_ratio_loss": 0.6954682469367981, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.09095387905836105, | |
| "rewards/margins": 0.009595084004104137, | |
| "rewards/rejected": -0.10054896771907806, | |
| "sft_loss": 0.909538745880127, | |
| "step": 760 | |
| }, | |
| { | |
| "epoch": 1.2446958981612446, | |
| "grad_norm": 0.3986392617225647, | |
| "learning_rate": 3.157510482110856e-06, | |
| "logits/chosen": -0.31712478399276733, | |
| "logits/rejected": -0.3332034647464752, | |
| "logps/chosen": -0.8950090408325195, | |
| "logps/rejected": -0.9677726626396179, | |
| "loss": 0.9687, | |
| "odds_ratio_loss": 0.7365735173225403, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.08950088918209076, | |
| "rewards/margins": 0.007276373915374279, | |
| "rewards/rejected": -0.09677727520465851, | |
| "sft_loss": 0.8950090408325195, | |
| "step": 770 | |
| }, | |
| { | |
| "epoch": 1.2608607799555465, | |
| "grad_norm": 0.9783799648284912, | |
| "learning_rate": 3.116544849436077e-06, | |
| "logits/chosen": -0.3367740213871002, | |
| "logits/rejected": -0.3552953600883484, | |
| "logps/chosen": -0.9589813351631165, | |
| "logps/rejected": -1.1763808727264404, | |
| "loss": 1.0263, | |
| "odds_ratio_loss": 0.6732120513916016, | |
| "rewards/accuracies": 0.4749999940395355, | |
| "rewards/chosen": -0.09589814394712448, | |
| "rewards/margins": 0.02173994854092598, | |
| "rewards/rejected": -0.11763808876276016, | |
| "sft_loss": 0.9589813351631165, | |
| "step": 780 | |
| }, | |
| { | |
| "epoch": 1.2770256617498483, | |
| "grad_norm": 0.3939819931983948, | |
| "learning_rate": 3.0754021726778848e-06, | |
| "logits/chosen": -0.3505743741989136, | |
| "logits/rejected": -0.37322431802749634, | |
| "logps/chosen": -0.83990079164505, | |
| "logps/rejected": -1.0232980251312256, | |
| "loss": 0.9049, | |
| "odds_ratio_loss": 0.6501890420913696, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.08399007469415665, | |
| "rewards/margins": 0.018339723348617554, | |
| "rewards/rejected": -0.1023297905921936, | |
| "sft_loss": 0.83990079164505, | |
| "step": 790 | |
| }, | |
| { | |
| "epoch": 1.2931905435441502, | |
| "grad_norm": 0.35344642400741577, | |
| "learning_rate": 3.0340942661714463e-06, | |
| "logits/chosen": -0.3435738980770111, | |
| "logits/rejected": -0.36761245131492615, | |
| "logps/chosen": -0.9316965341567993, | |
| "logps/rejected": -1.0095479488372803, | |
| "loss": 1.003, | |
| "odds_ratio_loss": 0.7125651836395264, | |
| "rewards/accuracies": 0.48750001192092896, | |
| "rewards/chosen": -0.09316965192556381, | |
| "rewards/margins": 0.007785154972225428, | |
| "rewards/rejected": -0.1009548082947731, | |
| "sft_loss": 0.9316965341567993, | |
| "step": 800 | |
| }, | |
| { | |
| "epoch": 1.3093554253384523, | |
| "grad_norm": 0.4086878001689911, | |
| "learning_rate": 2.992632991698512e-06, | |
| "logits/chosen": -0.39886465668678284, | |
| "logits/rejected": -0.3849073350429535, | |
| "logps/chosen": -0.9022181630134583, | |
| "logps/rejected": -1.0039399862289429, | |
| "loss": 0.9729, | |
| "odds_ratio_loss": 0.7066690325737, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.09022180736064911, | |
| "rewards/margins": 0.010172189213335514, | |
| "rewards/rejected": -0.10039399564266205, | |
| "sft_loss": 0.9022181630134583, | |
| "step": 810 | |
| }, | |
| { | |
| "epoch": 1.3255203071327541, | |
| "grad_norm": 0.45464497804641724, | |
| "learning_rate": 2.9510302550812537e-06, | |
| "logits/chosen": -0.3623855710029602, | |
| "logits/rejected": -0.31726986169815063, | |
| "logps/chosen": -0.8218330144882202, | |
| "logps/rejected": -1.0319081544876099, | |
| "loss": 0.8851, | |
| "odds_ratio_loss": 0.6329900026321411, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.08218331634998322, | |
| "rewards/margins": 0.02100750431418419, | |
| "rewards/rejected": -0.1031908169388771, | |
| "sft_loss": 0.8218330144882202, | |
| "step": 820 | |
| }, | |
| { | |
| "epoch": 1.341685188927056, | |
| "grad_norm": 1.1504096984863281, | |
| "learning_rate": 2.9092980027634325e-06, | |
| "logits/chosen": -0.38953226804733276, | |
| "logits/rejected": -0.3612954914569855, | |
| "logps/chosen": -0.8214972615242004, | |
| "logps/rejected": -0.9684427976608276, | |
| "loss": 0.8864, | |
| "odds_ratio_loss": 0.6492589712142944, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.08214972913265228, | |
| "rewards/margins": 0.014694547280669212, | |
| "rewards/rejected": -0.09684427082538605, | |
| "sft_loss": 0.8214972615242004, | |
| "step": 830 | |
| }, | |
| { | |
| "epoch": 1.3578500707213579, | |
| "grad_norm": 0.33391210436820984, | |
| "learning_rate": 2.867448218379927e-06, | |
| "logits/chosen": -0.3767167627811432, | |
| "logits/rejected": -0.3566213548183441, | |
| "logps/chosen": -0.9622126817703247, | |
| "logps/rejected": -1.030574083328247, | |
| "loss": 1.0363, | |
| "odds_ratio_loss": 0.7405400276184082, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.09622127562761307, | |
| "rewards/margins": 0.006836143787950277, | |
| "rewards/rejected": -0.10305740684270859, | |
| "sft_loss": 0.9622126817703247, | |
| "step": 840 | |
| }, | |
| { | |
| "epoch": 1.3740149525156597, | |
| "grad_norm": 1.2477465867996216, | |
| "learning_rate": 2.825492919315559e-06, | |
| "logits/chosen": -0.3341541886329651, | |
| "logits/rejected": -0.28563547134399414, | |
| "logps/chosen": -0.9898349046707153, | |
| "logps/rejected": -0.9626699686050415, | |
| "loss": 1.0687, | |
| "odds_ratio_loss": 0.7890844345092773, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.09898348897695541, | |
| "rewards/margins": -0.0027165021747350693, | |
| "rewards/rejected": -0.0962669849395752, | |
| "sft_loss": 0.9898349046707153, | |
| "step": 850 | |
| }, | |
| { | |
| "epoch": 1.3901798343099616, | |
| "grad_norm": 0.37100037932395935, | |
| "learning_rate": 2.7834441532542482e-06, | |
| "logits/chosen": -0.3620319366455078, | |
| "logits/rejected": -0.3429003357887268, | |
| "logps/chosen": -0.8693292737007141, | |
| "logps/rejected": -0.991874098777771, | |
| "loss": 0.9379, | |
| "odds_ratio_loss": 0.6856324076652527, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.08693292737007141, | |
| "rewards/margins": 0.012254483997821808, | |
| "rewards/rejected": -0.09918741136789322, | |
| "sft_loss": 0.8693292737007141, | |
| "step": 860 | |
| }, | |
| { | |
| "epoch": 1.4063447161042635, | |
| "grad_norm": 1.2096267938613892, | |
| "learning_rate": 2.74131399471945e-06, | |
| "logits/chosen": -0.3446846306324005, | |
| "logits/rejected": -0.3061850666999817, | |
| "logps/chosen": -0.9667361974716187, | |
| "logps/rejected": -1.053593397140503, | |
| "loss": 1.0382, | |
| "odds_ratio_loss": 0.714438796043396, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.0966736227273941, | |
| "rewards/margins": 0.008685723878443241, | |
| "rewards/rejected": -0.10535935312509537, | |
| "sft_loss": 0.9667361974716187, | |
| "step": 870 | |
| }, | |
| { | |
| "epoch": 1.4225095978985653, | |
| "grad_norm": 0.47893857955932617, | |
| "learning_rate": 2.6991145416068947e-06, | |
| "logits/chosen": -0.3955840468406677, | |
| "logits/rejected": -0.31594154238700867, | |
| "logps/chosen": -0.9019123315811157, | |
| "logps/rejected": -0.9488536715507507, | |
| "loss": 0.9734, | |
| "odds_ratio_loss": 0.7147491574287415, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.09019123762845993, | |
| "rewards/margins": 0.0046941377222537994, | |
| "rewards/rejected": -0.09488537907600403, | |
| "sft_loss": 0.9019123315811157, | |
| "step": 880 | |
| }, | |
| { | |
| "epoch": 1.4386744796928672, | |
| "grad_norm": 0.2868447005748749, | |
| "learning_rate": 2.6568579117106143e-06, | |
| "logits/chosen": -0.4024140238761902, | |
| "logits/rejected": -0.4033503532409668, | |
| "logps/chosen": -0.8388016819953918, | |
| "logps/rejected": -0.9728044271469116, | |
| "loss": 0.9081, | |
| "odds_ratio_loss": 0.6926370859146118, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.08388017117977142, | |
| "rewards/margins": 0.013400280848145485, | |
| "rewards/rejected": -0.09728045761585236, | |
| "sft_loss": 0.8388016819953918, | |
| "step": 890 | |
| }, | |
| { | |
| "epoch": 1.454839361487169, | |
| "grad_norm": 0.24462518095970154, | |
| "learning_rate": 2.6145562392432544e-06, | |
| "logits/chosen": -0.3949779272079468, | |
| "logits/rejected": -0.39668601751327515, | |
| "logps/chosen": -0.8613153696060181, | |
| "logps/rejected": -0.9795036315917969, | |
| "loss": 0.9305, | |
| "odds_ratio_loss": 0.6919496059417725, | |
| "rewards/accuracies": 0.4937500059604645, | |
| "rewards/chosen": -0.08613153547048569, | |
| "rewards/margins": 0.011818833649158478, | |
| "rewards/rejected": -0.09795036166906357, | |
| "sft_loss": 0.8613153696060181, | |
| "step": 900 | |
| }, | |
| { | |
| "epoch": 1.471004243281471, | |
| "grad_norm": 0.5152093768119812, | |
| "learning_rate": 2.5722216713516682e-06, | |
| "logits/chosen": -0.42058199644088745, | |
| "logits/rejected": -0.38909250497817993, | |
| "logps/chosen": -0.8609904050827026, | |
| "logps/rejected": -0.9690335988998413, | |
| "loss": 0.9318, | |
| "odds_ratio_loss": 0.7082633972167969, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.0860990509390831, | |
| "rewards/margins": 0.010804320685565472, | |
| "rewards/rejected": -0.09690337628126144, | |
| "sft_loss": 0.8609904050827026, | |
| "step": 910 | |
| }, | |
| { | |
| "epoch": 1.4871691250757728, | |
| "grad_norm": 0.5419692397117615, | |
| "learning_rate": 2.5298663646288064e-06, | |
| "logits/chosen": -0.35978519916534424, | |
| "logits/rejected": -0.35384541749954224, | |
| "logps/chosen": -0.8710163235664368, | |
| "logps/rejected": -1.0426474809646606, | |
| "loss": 0.9373, | |
| "odds_ratio_loss": 0.6623716354370117, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.08710163086652756, | |
| "rewards/margins": 0.017163105309009552, | |
| "rewards/rejected": -0.10426473617553711, | |
| "sft_loss": 0.8710163235664368, | |
| "step": 920 | |
| }, | |
| { | |
| "epoch": 1.503334006870075, | |
| "grad_norm": 3.1488473415374756, | |
| "learning_rate": 2.487502481622879e-06, | |
| "logits/chosen": -0.4146711230278015, | |
| "logits/rejected": -0.40715789794921875, | |
| "logps/chosen": -0.9579635858535767, | |
| "logps/rejected": -1.0180439949035645, | |
| "loss": 1.0298, | |
| "odds_ratio_loss": 0.718089759349823, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.0957963615655899, | |
| "rewards/margins": 0.006008026655763388, | |
| "rewards/rejected": -0.10180439800024033, | |
| "sft_loss": 0.9579635858535767, | |
| "step": 930 | |
| }, | |
| { | |
| "epoch": 1.5194988886643768, | |
| "grad_norm": 0.6520385146141052, | |
| "learning_rate": 2.4451421873448253e-06, | |
| "logits/chosen": -0.3981381952762604, | |
| "logits/rejected": -0.33850008249282837, | |
| "logps/chosen": -0.9044814109802246, | |
| "logps/rejected": -0.9930024147033691, | |
| "loss": 0.9767, | |
| "odds_ratio_loss": 0.7225072979927063, | |
| "rewards/accuracies": 0.48750001192092896, | |
| "rewards/chosen": -0.09044814109802246, | |
| "rewards/margins": 0.008852103725075722, | |
| "rewards/rejected": -0.09930024296045303, | |
| "sft_loss": 0.9044814109802246, | |
| "step": 940 | |
| }, | |
| { | |
| "epoch": 1.5356637704586786, | |
| "grad_norm": 0.5775251984596252, | |
| "learning_rate": 2.40279764577506e-06, | |
| "logits/chosen": -0.36691075563430786, | |
| "logits/rejected": -0.31715118885040283, | |
| "logps/chosen": -0.9193195104598999, | |
| "logps/rejected": -0.9655280113220215, | |
| "loss": 0.9919, | |
| "odds_ratio_loss": 0.7258428931236267, | |
| "rewards/accuracies": 0.48750001192092896, | |
| "rewards/chosen": -0.09193196147680283, | |
| "rewards/margins": 0.004620848223567009, | |
| "rewards/rejected": -0.09655280411243439, | |
| "sft_loss": 0.9193195104598999, | |
| "step": 950 | |
| }, | |
| { | |
| "epoch": 1.5518286522529805, | |
| "grad_norm": 0.4706912636756897, | |
| "learning_rate": 2.3604810163705242e-06, | |
| "logits/chosen": -0.3801175355911255, | |
| "logits/rejected": -0.34497779607772827, | |
| "logps/chosen": -0.8502659797668457, | |
| "logps/rejected": -0.9808200597763062, | |
| "loss": 0.9153, | |
| "odds_ratio_loss": 0.6503497362136841, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.08502660691738129, | |
| "rewards/margins": 0.013055416755378246, | |
| "rewards/rejected": -0.09808202087879181, | |
| "sft_loss": 0.8502659797668457, | |
| "step": 960 | |
| }, | |
| { | |
| "epoch": 1.5679935340472824, | |
| "grad_norm": 0.8772755265235901, | |
| "learning_rate": 2.3182044505730364e-06, | |
| "logits/chosen": -0.3701505661010742, | |
| "logits/rejected": -0.3588781952857971, | |
| "logps/chosen": -0.8278260231018066, | |
| "logps/rejected": -0.9880140423774719, | |
| "loss": 0.8943, | |
| "odds_ratio_loss": 0.6643026471138, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.08278260380029678, | |
| "rewards/margins": 0.016018804162740707, | |
| "rewards/rejected": -0.09880141168832779, | |
| "sft_loss": 0.8278260231018066, | |
| "step": 970 | |
| }, | |
| { | |
| "epoch": 1.5841584158415842, | |
| "grad_norm": 0.5644322633743286, | |
| "learning_rate": 2.275980088319941e-06, | |
| "logits/chosen": -0.37429267168045044, | |
| "logits/rejected": -0.38965049386024475, | |
| "logps/chosen": -0.830912709236145, | |
| "logps/rejected": -0.931898295879364, | |
| "loss": 0.901, | |
| "odds_ratio_loss": 0.7011361122131348, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.08309127390384674, | |
| "rewards/margins": 0.010098553262650967, | |
| "rewards/rejected": -0.09318983554840088, | |
| "sft_loss": 0.830912709236145, | |
| "step": 980 | |
| }, | |
| { | |
| "epoch": 1.600323297635886, | |
| "grad_norm": 0.7188877463340759, | |
| "learning_rate": 2.2338200545580577e-06, | |
| "logits/chosen": -0.387838214635849, | |
| "logits/rejected": -0.3446332514286041, | |
| "logps/chosen": -0.8468879461288452, | |
| "logps/rejected": -1.0357553958892822, | |
| "loss": 0.9171, | |
| "odds_ratio_loss": 0.7018327713012695, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.08468880504369736, | |
| "rewards/margins": 0.018886741250753403, | |
| "rewards/rejected": -0.10357554256916046, | |
| "sft_loss": 0.8468879461288452, | |
| "step": 990 | |
| }, | |
| { | |
| "epoch": 1.616488179430188, | |
| "grad_norm": 0.40455734729766846, | |
| "learning_rate": 2.191736455761947e-06, | |
| "logits/chosen": -0.32430940866470337, | |
| "logits/rejected": -0.3191392719745636, | |
| "logps/chosen": -0.7817317247390747, | |
| "logps/rejected": -0.8739973306655884, | |
| "loss": 0.8458, | |
| "odds_ratio_loss": 0.6406995058059692, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.07817317545413971, | |
| "rewards/margins": 0.009226562455296516, | |
| "rewards/rejected": -0.08739973604679108, | |
| "sft_loss": 0.7817317247390747, | |
| "step": 1000 | |
| }, | |
| { | |
| "epoch": 1.616488179430188, | |
| "eval_logits/chosen": -0.3771926760673523, | |
| "eval_logits/rejected": -0.3578239679336548, | |
| "eval_logps/chosen": -0.8850269317626953, | |
| "eval_logps/rejected": -0.9999891519546509, | |
| "eval_loss": 0.9560017585754395, | |
| "eval_odds_ratio_loss": 0.7097483277320862, | |
| "eval_rewards/accuracies": 0.5190908908843994, | |
| "eval_rewards/chosen": -0.08850269019603729, | |
| "eval_rewards/margins": 0.0114962263032794, | |
| "eval_rewards/rejected": -0.09999892115592957, | |
| "eval_runtime": 192.1461, | |
| "eval_samples_per_second": 5.725, | |
| "eval_sft_loss": 0.8850269317626953, | |
| "eval_steps_per_second": 2.862, | |
| "step": 1000 | |
| }, | |
| { | |
| "epoch": 1.6326530612244898, | |
| "grad_norm": 0.3581576347351074, | |
| "learning_rate": 2.1497413764574673e-06, | |
| "logits/chosen": -0.31272074580192566, | |
| "logits/rejected": -0.33244556188583374, | |
| "logps/chosen": -0.9109123349189758, | |
| "logps/rejected": -1.0614047050476074, | |
| "loss": 0.9769, | |
| "odds_ratio_loss": 0.6601108908653259, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.09109123051166534, | |
| "rewards/margins": 0.01504923403263092, | |
| "rewards/rejected": -0.10614047199487686, | |
| "sft_loss": 0.9109123349189758, | |
| "step": 1010 | |
| }, | |
| { | |
| "epoch": 1.6488179430187917, | |
| "grad_norm": 1.0781522989273071, | |
| "learning_rate": 2.1078468757516395e-06, | |
| "logits/chosen": -0.3577747941017151, | |
| "logits/rejected": -0.372037798166275, | |
| "logps/chosen": -0.8666743040084839, | |
| "logps/rejected": -0.9286467432975769, | |
| "loss": 0.943, | |
| "odds_ratio_loss": 0.7631633877754211, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.08666743338108063, | |
| "rewards/margins": 0.006197246722877026, | |
| "rewards/rejected": -0.09286467730998993, | |
| "sft_loss": 0.8666743040084839, | |
| "step": 1020 | |
| }, | |
| { | |
| "epoch": 1.6649828248130936, | |
| "grad_norm": 0.4093440771102905, | |
| "learning_rate": 2.0660649838698145e-06, | |
| "logits/chosen": -0.24239635467529297, | |
| "logits/rejected": -0.2550283670425415, | |
| "logps/chosen": -0.8779211044311523, | |
| "logps/rejected": -1.028240442276001, | |
| "loss": 0.9471, | |
| "odds_ratio_loss": 0.691811203956604, | |
| "rewards/accuracies": 0.46875, | |
| "rewards/chosen": -0.08779212832450867, | |
| "rewards/margins": 0.015031938441097736, | |
| "rewards/rejected": -0.10282406955957413, | |
| "sft_loss": 0.8779211044311523, | |
| "step": 1030 | |
| }, | |
| { | |
| "epoch": 1.6811477066073954, | |
| "grad_norm": 0.4143465459346771, | |
| "learning_rate": 2.0244076987011284e-06, | |
| "logits/chosen": -0.320882648229599, | |
| "logits/rejected": -0.35348570346832275, | |
| "logps/chosen": -0.9102975726127625, | |
| "logps/rejected": -1.0311200618743896, | |
| "loss": 0.9776, | |
| "odds_ratio_loss": 0.6728986501693726, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.09102976322174072, | |
| "rewards/margins": 0.012082245200872421, | |
| "rewards/rejected": -0.10311201959848404, | |
| "sft_loss": 0.9102975726127625, | |
| "step": 1040 | |
| }, | |
| { | |
| "epoch": 1.6973125884016973, | |
| "grad_norm": 0.4322679340839386, | |
| "learning_rate": 1.982886982353251e-06, | |
| "logits/chosen": -0.33857375383377075, | |
| "logits/rejected": -0.38647031784057617, | |
| "logps/chosen": -0.8801182508468628, | |
| "logps/rejected": -1.0462461709976196, | |
| "loss": 0.9472, | |
| "odds_ratio_loss": 0.6703814268112183, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.08801182359457016, | |
| "rewards/margins": 0.016612788662314415, | |
| "rewards/rejected": -0.10462461411952972, | |
| "sft_loss": 0.8801182508468628, | |
| "step": 1050 | |
| }, | |
| { | |
| "epoch": 1.7134774701959992, | |
| "grad_norm": 0.40310564637184143, | |
| "learning_rate": 1.941514757717392e-06, | |
| "logits/chosen": -0.3961712718009949, | |
| "logits/rejected": -0.3599357604980469, | |
| "logps/chosen": -0.857568621635437, | |
| "logps/rejected": -1.0133601427078247, | |
| "loss": 0.921, | |
| "odds_ratio_loss": 0.6347678899765015, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.08575686812400818, | |
| "rewards/margins": 0.015579144470393658, | |
| "rewards/rejected": -0.10133601725101471, | |
| "sft_loss": 0.857568621635437, | |
| "step": 1060 | |
| }, | |
| { | |
| "epoch": 1.729642351990301, | |
| "grad_norm": 0.5565314888954163, | |
| "learning_rate": 1.9003029050445953e-06, | |
| "logits/chosen": -0.3478461802005768, | |
| "logits/rejected": -0.3207647204399109, | |
| "logps/chosen": -0.9041654467582703, | |
| "logps/rejected": -0.99024897813797, | |
| "loss": 0.9734, | |
| "odds_ratio_loss": 0.6924456357955933, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.0904165506362915, | |
| "rewards/margins": 0.008608358912169933, | |
| "rewards/rejected": -0.09902490675449371, | |
| "sft_loss": 0.9041654467582703, | |
| "step": 1070 | |
| }, | |
| { | |
| "epoch": 1.745807233784603, | |
| "grad_norm": 0.4490904211997986, | |
| "learning_rate": 1.8592632585342523e-06, | |
| "logits/chosen": -0.36072981357574463, | |
| "logits/rejected": -0.3492718040943146, | |
| "logps/chosen": -0.8714792132377625, | |
| "logps/rejected": -1.010517954826355, | |
| "loss": 0.9396, | |
| "odds_ratio_loss": 0.6810620427131653, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.08714792132377625, | |
| "rewards/margins": 0.013903876766562462, | |
| "rewards/rejected": -0.10105180740356445, | |
| "sft_loss": 0.8714792132377625, | |
| "step": 1080 | |
| }, | |
| { | |
| "epoch": 1.7619721155789048, | |
| "grad_norm": 0.71334308385849, | |
| "learning_rate": 1.8184076029358527e-06, | |
| "logits/chosen": -0.3724268078804016, | |
| "logits/rejected": -0.40728870034217834, | |
| "logps/chosen": -0.8329513669013977, | |
| "logps/rejected": -0.8585556745529175, | |
| "loss": 0.9053, | |
| "odds_ratio_loss": 0.723603367805481, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.08329514414072037, | |
| "rewards/margins": 0.002560428809374571, | |
| "rewards/rejected": -0.08585558086633682, | |
| "sft_loss": 0.8329513669013977, | |
| "step": 1090 | |
| }, | |
| { | |
| "epoch": 1.7781369973732066, | |
| "grad_norm": 0.38024160265922546, | |
| "learning_rate": 1.7777476701649318e-06, | |
| "logits/chosen": -0.4104040563106537, | |
| "logits/rejected": -0.40031394362449646, | |
| "logps/chosen": -0.9076647758483887, | |
| "logps/rejected": -1.019285798072815, | |
| "loss": 0.9752, | |
| "odds_ratio_loss": 0.6755737662315369, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.09076648205518723, | |
| "rewards/margins": 0.011162097565829754, | |
| "rewards/rejected": -0.10192857682704926, | |
| "sft_loss": 0.9076647758483887, | |
| "step": 1100 | |
| }, | |
| { | |
| "epoch": 1.7943018791675085, | |
| "grad_norm": 0.433108389377594, | |
| "learning_rate": 1.7372951359341925e-06, | |
| "logits/chosen": -0.35082167387008667, | |
| "logits/rejected": -0.3622151017189026, | |
| "logps/chosen": -0.8306609988212585, | |
| "logps/rejected": -0.9346961975097656, | |
| "loss": 0.8994, | |
| "odds_ratio_loss": 0.6869168281555176, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.08306611329317093, | |
| "rewards/margins": 0.010403511114418507, | |
| "rewards/rejected": -0.09346961975097656, | |
| "sft_loss": 0.8306609988212585, | |
| "step": 1110 | |
| }, | |
| { | |
| "epoch": 1.8104667609618104, | |
| "grad_norm": 1.0182783603668213, | |
| "learning_rate": 1.6970616164007547e-06, | |
| "logits/chosen": -0.4078885614871979, | |
| "logits/rejected": -0.43148526549339294, | |
| "logps/chosen": -0.8258237838745117, | |
| "logps/rejected": -0.9274940490722656, | |
| "loss": 0.8967, | |
| "odds_ratio_loss": 0.7091785073280334, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.08258237689733505, | |
| "rewards/margins": 0.010167025960981846, | |
| "rewards/rejected": -0.09274940937757492, | |
| "sft_loss": 0.8258237838745117, | |
| "step": 1120 | |
| }, | |
| { | |
| "epoch": 1.8266316427561122, | |
| "grad_norm": 1.0357805490493774, | |
| "learning_rate": 1.6570586648305276e-06, | |
| "logits/chosen": -0.4377085268497467, | |
| "logits/rejected": -0.407601922750473, | |
| "logps/chosen": -0.8756824731826782, | |
| "logps/rejected": -1.0340659618377686, | |
| "loss": 0.9437, | |
| "odds_ratio_loss": 0.6799197793006897, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.0875682383775711, | |
| "rewards/margins": 0.01583835855126381, | |
| "rewards/rejected": -0.10340659320354462, | |
| "sft_loss": 0.8756824731826782, | |
| "step": 1130 | |
| }, | |
| { | |
| "epoch": 1.842796524550414, | |
| "grad_norm": 0.4784797430038452, | |
| "learning_rate": 1.6172977682806151e-06, | |
| "logits/chosen": -0.3374441862106323, | |
| "logits/rejected": -0.2926723062992096, | |
| "logps/chosen": -0.8671070337295532, | |
| "logps/rejected": -1.0173355340957642, | |
| "loss": 0.9326, | |
| "odds_ratio_loss": 0.6546159982681274, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.08671069890260696, | |
| "rewards/margins": 0.015022864565253258, | |
| "rewards/rejected": -0.10173355042934418, | |
| "sft_loss": 0.8671070337295532, | |
| "step": 1140 | |
| }, | |
| { | |
| "epoch": 1.858961406344716, | |
| "grad_norm": 0.5492507219314575, | |
| "learning_rate": 1.5777903443007586e-06, | |
| "logits/chosen": -0.3145988881587982, | |
| "logits/rejected": -0.42871540784835815, | |
| "logps/chosen": -0.8989070057868958, | |
| "logps/rejected": -1.0172455310821533, | |
| "loss": 0.9689, | |
| "odds_ratio_loss": 0.6998150944709778, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.08989070355892181, | |
| "rewards/margins": 0.011833854019641876, | |
| "rewards/rejected": -0.1017245501279831, | |
| "sft_loss": 0.8989070057868958, | |
| "step": 1150 | |
| }, | |
| { | |
| "epoch": 1.8751262881390178, | |
| "grad_norm": 0.4275898039340973, | |
| "learning_rate": 1.5385477376547226e-06, | |
| "logits/chosen": -0.3347630202770233, | |
| "logits/rejected": -0.34142249822616577, | |
| "logps/chosen": -0.9212555885314941, | |
| "logps/rejected": -1.0021544694900513, | |
| "loss": 0.9893, | |
| "odds_ratio_loss": 0.679952085018158, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.0921255499124527, | |
| "rewards/margins": 0.008089900948107243, | |
| "rewards/rejected": -0.10021545737981796, | |
| "sft_loss": 0.9212555885314941, | |
| "step": 1160 | |
| }, | |
| { | |
| "epoch": 1.89129116993332, | |
| "grad_norm": 0.5769237875938416, | |
| "learning_rate": 1.4995812170625845e-06, | |
| "logits/chosen": -0.3509088456630707, | |
| "logits/rejected": -0.35828500986099243, | |
| "logps/chosen": -0.8898354768753052, | |
| "logps/rejected": -1.1126220226287842, | |
| "loss": 0.9543, | |
| "odds_ratio_loss": 0.6445311307907104, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.08898355811834335, | |
| "rewards/margins": 0.02227865532040596, | |
| "rewards/rejected": -0.11126221716403961, | |
| "sft_loss": 0.8898354768753052, | |
| "step": 1170 | |
| }, | |
| { | |
| "epoch": 1.9074560517276218, | |
| "grad_norm": 0.9893414974212646, | |
| "learning_rate": 1.4609019719648666e-06, | |
| "logits/chosen": -0.34388267993927, | |
| "logits/rejected": -0.34255415201187134, | |
| "logps/chosen": -0.9129988551139832, | |
| "logps/rejected": -1.0511752367019653, | |
| "loss": 0.9778, | |
| "odds_ratio_loss": 0.6484531760215759, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.0912998765707016, | |
| "rewards/margins": 0.013817653059959412, | |
| "rewards/rejected": -0.10511753708124161, | |
| "sft_loss": 0.9129988551139832, | |
| "step": 1180 | |
| }, | |
| { | |
| "epoch": 1.9236209335219236, | |
| "grad_norm": 0.8161694407463074, | |
| "learning_rate": 1.42252110930943e-06, | |
| "logits/chosen": -0.3889426589012146, | |
| "logits/rejected": -0.37780189514160156, | |
| "logps/chosen": -0.8312114477157593, | |
| "logps/rejected": -0.9597098231315613, | |
| "loss": 0.8972, | |
| "odds_ratio_loss": 0.6594355702400208, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.08312113583087921, | |
| "rewards/margins": 0.01284984964877367, | |
| "rewards/rejected": -0.0959709957242012, | |
| "sft_loss": 0.8312114477157593, | |
| "step": 1190 | |
| }, | |
| { | |
| "epoch": 1.9397858153162255, | |
| "grad_norm": 0.6737188100814819, | |
| "learning_rate": 1.3844496503620493e-06, | |
| "logits/chosen": -0.34721988439559937, | |
| "logits/rejected": -0.29065969586372375, | |
| "logps/chosen": -0.8556321263313293, | |
| "logps/rejected": -0.9435693621635437, | |
| "loss": 0.9217, | |
| "odds_ratio_loss": 0.6608615517616272, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.08556319773197174, | |
| "rewards/margins": 0.008793738670647144, | |
| "rewards/rejected": -0.09435693919658661, | |
| "sft_loss": 0.8556321263313293, | |
| "step": 1200 | |
| }, | |
| { | |
| "epoch": 1.9559506971105274, | |
| "grad_norm": 1.0895054340362549, | |
| "learning_rate": 1.3466985275416081e-06, | |
| "logits/chosen": -0.38311949372291565, | |
| "logits/rejected": -0.440490186214447, | |
| "logps/chosen": -0.9350228309631348, | |
| "logps/rejected": -1.0175323486328125, | |
| "loss": 1.0086, | |
| "odds_ratio_loss": 0.7355881929397583, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.09350229054689407, | |
| "rewards/margins": 0.008250946179032326, | |
| "rewards/rejected": -0.10175323486328125, | |
| "sft_loss": 0.9350228309631348, | |
| "step": 1210 | |
| }, | |
| { | |
| "epoch": 1.9721155789048292, | |
| "grad_norm": 0.7546266913414001, | |
| "learning_rate": 1.309278581280791e-06, | |
| "logits/chosen": -0.32461339235305786, | |
| "logits/rejected": -0.38296985626220703, | |
| "logps/chosen": -0.825161337852478, | |
| "logps/rejected": -1.007612943649292, | |
| "loss": 0.8897, | |
| "odds_ratio_loss": 0.6452582478523254, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.082516148686409, | |
| "rewards/margins": 0.018245156854391098, | |
| "rewards/rejected": -0.1007612943649292, | |
| "sft_loss": 0.825161337852478, | |
| "step": 1220 | |
| }, | |
| { | |
| "epoch": 1.9882804606991311, | |
| "grad_norm": 0.30651387572288513, | |
| "learning_rate": 1.272200556913199e-06, | |
| "logits/chosen": -0.34240493178367615, | |
| "logits/rejected": -0.33365195989608765, | |
| "logps/chosen": -0.9005836248397827, | |
| "logps/rejected": -1.0132153034210205, | |
| "loss": 0.9729, | |
| "odds_ratio_loss": 0.722726583480835, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.09005837142467499, | |
| "rewards/margins": 0.011263175867497921, | |
| "rewards/rejected": -0.10132155567407608, | |
| "sft_loss": 0.9005836248397827, | |
| "step": 1230 | |
| }, | |
| { | |
| "epoch": 2.004445342493433, | |
| "grad_norm": 0.6191690564155579, | |
| "learning_rate": 1.2354751015877698e-06, | |
| "logits/chosen": -0.3653295636177063, | |
| "logits/rejected": -0.3104439675807953, | |
| "logps/chosen": -0.8316798210144043, | |
| "logps/rejected": -1.0361697673797607, | |
| "loss": 0.8947, | |
| "odds_ratio_loss": 0.6298761963844299, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.08316798508167267, | |
| "rewards/margins": 0.02044900692999363, | |
| "rewards/rejected": -0.10361699759960175, | |
| "sft_loss": 0.8316798210144043, | |
| "step": 1240 | |
| }, | |
| { | |
| "epoch": 2.020610224287735, | |
| "grad_norm": 0.752289354801178, | |
| "learning_rate": 1.1991127612113945e-06, | |
| "logits/chosen": -0.3582732379436493, | |
| "logits/rejected": -0.3034323751926422, | |
| "logps/chosen": -0.8952615857124329, | |
| "logps/rejected": -1.0291544198989868, | |
| "loss": 0.9609, | |
| "odds_ratio_loss": 0.6566318869590759, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.08952615410089493, | |
| "rewards/margins": 0.013389283791184425, | |
| "rewards/rejected": -0.1029154434800148, | |
| "sft_loss": 0.8952615857124329, | |
| "step": 1250 | |
| }, | |
| { | |
| "epoch": 2.036775106082037, | |
| "grad_norm": 0.5910158753395081, | |
| "learning_rate": 1.1631239774206035e-06, | |
| "logits/chosen": -0.36862578988075256, | |
| "logits/rejected": -0.3653218150138855, | |
| "logps/chosen": -0.8613477945327759, | |
| "logps/rejected": -0.9755401611328125, | |
| "loss": 0.9325, | |
| "odds_ratio_loss": 0.7117538452148438, | |
| "rewards/accuracies": 0.46875, | |
| "rewards/chosen": -0.08613476902246475, | |
| "rewards/margins": 0.011419234797358513, | |
| "rewards/rejected": -0.09755401313304901, | |
| "sft_loss": 0.8613477945327759, | |
| "step": 1260 | |
| }, | |
| { | |
| "epoch": 2.052939987876339, | |
| "grad_norm": 0.5977714657783508, | |
| "learning_rate": 1.1275190845831978e-06, | |
| "logits/chosen": -0.35793787240982056, | |
| "logits/rejected": -0.3579494059085846, | |
| "logps/chosen": -0.8839446902275085, | |
| "logps/rejected": -1.0555723905563354, | |
| "loss": 0.9484, | |
| "odds_ratio_loss": 0.6443823575973511, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.08839447796344757, | |
| "rewards/margins": 0.01716277375817299, | |
| "rewards/rejected": -0.10555724799633026, | |
| "sft_loss": 0.8839446902275085, | |
| "step": 1270 | |
| }, | |
| { | |
| "epoch": 2.0691048696706407, | |
| "grad_norm": 0.4356369078159332, | |
| "learning_rate": 1.0923083068306778e-06, | |
| "logits/chosen": -0.2889194190502167, | |
| "logits/rejected": -0.39258915185928345, | |
| "logps/chosen": -0.8745051622390747, | |
| "logps/rejected": -1.061402678489685, | |
| "loss": 0.94, | |
| "odds_ratio_loss": 0.6551867723464966, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.08745051920413971, | |
| "rewards/margins": 0.018689759075641632, | |
| "rewards/rejected": -0.10614027827978134, | |
| "sft_loss": 0.8745051622390747, | |
| "step": 1280 | |
| }, | |
| { | |
| "epoch": 2.0852697514649425, | |
| "grad_norm": 0.2981340289115906, | |
| "learning_rate": 1.0575017551223348e-06, | |
| "logits/chosen": -0.39015138149261475, | |
| "logits/rejected": -0.40903449058532715, | |
| "logps/chosen": -0.7750725746154785, | |
| "logps/rejected": -0.9115964770317078, | |
| "loss": 0.8412, | |
| "odds_ratio_loss": 0.6609454154968262, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.07750725001096725, | |
| "rewards/margins": 0.013652404770255089, | |
| "rewards/rejected": -0.09115965664386749, | |
| "sft_loss": 0.7750725746154785, | |
| "step": 1290 | |
| }, | |
| { | |
| "epoch": 2.1014346332592444, | |
| "grad_norm": 0.39186251163482666, | |
| "learning_rate": 1.023109424341833e-06, | |
| "logits/chosen": -0.3986419141292572, | |
| "logits/rejected": -0.36254242062568665, | |
| "logps/chosen": -0.8747810125350952, | |
| "logps/rejected": -0.9971181750297546, | |
| "loss": 0.9444, | |
| "odds_ratio_loss": 0.6959220170974731, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.08747810870409012, | |
| "rewards/margins": 0.012233709916472435, | |
| "rewards/rejected": -0.0997118204832077, | |
| "sft_loss": 0.8747810125350952, | |
| "step": 1300 | |
| }, | |
| { | |
| "epoch": 2.1175995150535463, | |
| "grad_norm": 0.4826388359069824, | |
| "learning_rate": 9.891411904271273e-07, | |
| "logits/chosen": -0.3570977747440338, | |
| "logits/rejected": -0.34066206216812134, | |
| "logps/chosen": -0.8385666608810425, | |
| "logps/rejected": -0.9865023493766785, | |
| "loss": 0.9076, | |
| "odds_ratio_loss": 0.6902373433113098, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.08385667204856873, | |
| "rewards/margins": 0.014793576672673225, | |
| "rewards/rejected": -0.0986502468585968, | |
| "sft_loss": 0.8385666608810425, | |
| "step": 1310 | |
| }, | |
| { | |
| "epoch": 2.133764396847848, | |
| "grad_norm": 0.3553561866283417, | |
| "learning_rate": 9.556068075345363e-07, | |
| "logits/chosen": -0.28917670249938965, | |
| "logits/rejected": -0.3470838665962219, | |
| "logps/chosen": -0.8463741540908813, | |
| "logps/rejected": -0.9492172002792358, | |
| "loss": 0.9162, | |
| "odds_ratio_loss": 0.6985523104667664, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.08463741838932037, | |
| "rewards/margins": 0.01028431672602892, | |
| "rewards/rejected": -0.09492173045873642, | |
| "sft_loss": 0.8463741540908813, | |
| "step": 1320 | |
| }, | |
| { | |
| "epoch": 2.14992927864215, | |
| "grad_norm": 0.3664523959159851, | |
| "learning_rate": 9.225159052377838e-07, | |
| "logits/chosen": -0.3276691436767578, | |
| "logits/rejected": -0.3102811872959137, | |
| "logps/chosen": -0.9000975489616394, | |
| "logps/rejected": -1.0900113582611084, | |
| "loss": 0.9658, | |
| "odds_ratio_loss": 0.6572277545928955, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.09000976383686066, | |
| "rewards/margins": 0.018991392105817795, | |
| "rewards/rejected": -0.10900114476680756, | |
| "sft_loss": 0.9000975489616394, | |
| "step": 1330 | |
| }, | |
| { | |
| "epoch": 2.166094160436452, | |
| "grad_norm": 0.5697169899940491, | |
| "learning_rate": 8.898779857628184e-07, | |
| "logits/chosen": -0.35697469115257263, | |
| "logits/rejected": -0.29451218247413635, | |
| "logps/chosen": -0.7642744779586792, | |
| "logps/rejected": -0.8856114149093628, | |
| "loss": 0.8306, | |
| "odds_ratio_loss": 0.6628420948982239, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.07642744481563568, | |
| "rewards/margins": 0.012133700773119926, | |
| "rewards/rejected": -0.08856116235256195, | |
| "sft_loss": 0.7642744779586792, | |
| "step": 1340 | |
| }, | |
| { | |
| "epoch": 2.1822590422307537, | |
| "grad_norm": 1.7151192426681519, | |
| "learning_rate": 8.577024212591975e-07, | |
| "logits/chosen": -0.29253047704696655, | |
| "logits/rejected": -0.3413800001144409, | |
| "logps/chosen": -0.8930098414421082, | |
| "logps/rejected": -0.9748668670654297, | |
| "loss": 0.9639, | |
| "odds_ratio_loss": 0.708949089050293, | |
| "rewards/accuracies": 0.4937500059604645, | |
| "rewards/chosen": -0.0893009752035141, | |
| "rewards/margins": 0.008185721933841705, | |
| "rewards/rejected": -0.09748668968677521, | |
| "sft_loss": 0.8930098414421082, | |
| "step": 1350 | |
| }, | |
| { | |
| "epoch": 2.1984239240250556, | |
| "grad_norm": 0.49061620235443115, | |
| "learning_rate": 8.259984511088276e-07, | |
| "logits/chosen": -0.3223104476928711, | |
| "logits/rejected": -0.29760584235191345, | |
| "logps/chosen": -0.8736541867256165, | |
| "logps/rejected": -0.9874069094657898, | |
| "loss": 0.9451, | |
| "odds_ratio_loss": 0.7148812413215637, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.08736542612314224, | |
| "rewards/margins": 0.011375268921256065, | |
| "rewards/rejected": -0.09874069690704346, | |
| "sft_loss": 0.8736541867256165, | |
| "step": 1360 | |
| }, | |
| { | |
| "epoch": 2.2145888058193575, | |
| "grad_norm": 0.33556151390075684, | |
| "learning_rate": 7.947751792728237e-07, | |
| "logits/chosen": -0.3239595890045166, | |
| "logits/rejected": -0.34610220789909363, | |
| "logps/chosen": -0.8864496946334839, | |
| "logps/rejected": -1.0747450590133667, | |
| "loss": 0.9559, | |
| "odds_ratio_loss": 0.694658637046814, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.08864498138427734, | |
| "rewards/margins": 0.018829550594091415, | |
| "rewards/rejected": -0.10747452825307846, | |
| "sft_loss": 0.8864496946334839, | |
| "step": 1370 | |
| }, | |
| { | |
| "epoch": 2.2307536876136593, | |
| "grad_norm": 0.5993340611457825, | |
| "learning_rate": 7.640415716772626e-07, | |
| "logits/chosen": -0.3385930359363556, | |
| "logits/rejected": -0.31589871644973755, | |
| "logps/chosen": -0.8884540796279907, | |
| "logps/rejected": -1.0432296991348267, | |
| "loss": 0.9579, | |
| "odds_ratio_loss": 0.6948095560073853, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.08884540945291519, | |
| "rewards/margins": 0.015477565117180347, | |
| "rewards/rejected": -0.10432296991348267, | |
| "sft_loss": 0.8884540796279907, | |
| "step": 1380 | |
| }, | |
| { | |
| "epoch": 2.246918569407961, | |
| "grad_norm": 0.4777003228664398, | |
| "learning_rate": 7.338064536385722e-07, | |
| "logits/chosen": -0.3243527412414551, | |
| "logits/rejected": -0.3211807608604431, | |
| "logps/chosen": -0.8481816053390503, | |
| "logps/rejected": -1.0429704189300537, | |
| "loss": 0.9136, | |
| "odds_ratio_loss": 0.6539761424064636, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.08481816202402115, | |
| "rewards/margins": 0.019478868693113327, | |
| "rewards/rejected": -0.10429704189300537, | |
| "sft_loss": 0.8481816053390503, | |
| "step": 1390 | |
| }, | |
| { | |
| "epoch": 2.263083451202263, | |
| "grad_norm": 0.6625237464904785, | |
| "learning_rate": 7.040785073292883e-07, | |
| "logits/chosen": -0.39626187086105347, | |
| "logits/rejected": -0.3658468425273895, | |
| "logps/chosen": -0.9418588876724243, | |
| "logps/rejected": -1.017301321029663, | |
| "loss": 1.0184, | |
| "odds_ratio_loss": 0.7650783658027649, | |
| "rewards/accuracies": 0.48750001192092896, | |
| "rewards/chosen": -0.09418588131666183, | |
| "rewards/margins": 0.007544253021478653, | |
| "rewards/rejected": -0.10173014551401138, | |
| "sft_loss": 0.9418588876724243, | |
| "step": 1400 | |
| }, | |
| { | |
| "epoch": 2.279248332996565, | |
| "grad_norm": 0.5683190226554871, | |
| "learning_rate": 6.748662692849297e-07, | |
| "logits/chosen": -0.2916708290576935, | |
| "logits/rejected": -0.289817750453949, | |
| "logps/chosen": -0.8634734153747559, | |
| "logps/rejected": -1.1026208400726318, | |
| "loss": 0.9275, | |
| "odds_ratio_loss": 0.6400235295295715, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.08634734898805618, | |
| "rewards/margins": 0.02391473576426506, | |
| "rewards/rejected": -0.11026208102703094, | |
| "sft_loss": 0.8634734153747559, | |
| "step": 1410 | |
| }, | |
| { | |
| "epoch": 2.295413214790867, | |
| "grad_norm": 1.625442624092102, | |
| "learning_rate": 6.46178127952686e-07, | |
| "logits/chosen": -0.35586509108543396, | |
| "logits/rejected": -0.35335296392440796, | |
| "logps/chosen": -0.8400161862373352, | |
| "logps/rejected": -0.9910812377929688, | |
| "loss": 0.9025, | |
| "odds_ratio_loss": 0.6247957348823547, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.08400160819292068, | |
| "rewards/margins": 0.015106521546840668, | |
| "rewards/rejected": -0.09910812973976135, | |
| "sft_loss": 0.8400161862373352, | |
| "step": 1420 | |
| }, | |
| { | |
| "epoch": 2.3115780965851687, | |
| "grad_norm": 0.46490368247032166, | |
| "learning_rate": 6.180223212826289e-07, | |
| "logits/chosen": -0.33770841360092163, | |
| "logits/rejected": -0.37226027250289917, | |
| "logps/chosen": -0.858726978302002, | |
| "logps/rejected": -0.9763249158859253, | |
| "loss": 0.9249, | |
| "odds_ratio_loss": 0.6612924933433533, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.08587270230054855, | |
| "rewards/margins": 0.011759791523218155, | |
| "rewards/rejected": -0.097632497549057, | |
| "sft_loss": 0.858726978302002, | |
| "step": 1430 | |
| }, | |
| { | |
| "epoch": 2.3277429783794705, | |
| "grad_norm": 0.25405463576316833, | |
| "learning_rate": 5.904069343621443e-07, | |
| "logits/chosen": -0.3201651871204376, | |
| "logits/rejected": -0.34286874532699585, | |
| "logps/chosen": -0.9113739132881165, | |
| "logps/rejected": -1.0487134456634521, | |
| "loss": 0.9766, | |
| "odds_ratio_loss": 0.652290403842926, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.09113740175962448, | |
| "rewards/margins": 0.013733962550759315, | |
| "rewards/rejected": -0.10487135499715805, | |
| "sft_loss": 0.9113739132881165, | |
| "step": 1440 | |
| }, | |
| { | |
| "epoch": 2.3439078601737724, | |
| "grad_norm": 0.5318045020103455, | |
| "learning_rate": 5.633398970942544e-07, | |
| "logits/chosen": -0.32512596249580383, | |
| "logits/rejected": -0.2820747494697571, | |
| "logps/chosen": -0.8218180537223816, | |
| "logps/rejected": -0.9094691276550293, | |
| "loss": 0.8927, | |
| "odds_ratio_loss": 0.7083881497383118, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.08218181133270264, | |
| "rewards/margins": 0.008765103295445442, | |
| "rewards/rejected": -0.09094691276550293, | |
| "sft_loss": 0.8218180537223816, | |
| "step": 1450 | |
| }, | |
| { | |
| "epoch": 2.3600727419680743, | |
| "grad_norm": 0.6843146681785583, | |
| "learning_rate": 5.368289819205069e-07, | |
| "logits/chosen": -0.39002543687820435, | |
| "logits/rejected": -0.376250684261322, | |
| "logps/chosen": -0.7933530211448669, | |
| "logps/rejected": -0.9611787796020508, | |
| "loss": 0.8585, | |
| "odds_ratio_loss": 0.6519256234169006, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.0793353021144867, | |
| "rewards/margins": 0.016782574355602264, | |
| "rewards/rejected": -0.09611787647008896, | |
| "sft_loss": 0.7933530211448669, | |
| "step": 1460 | |
| }, | |
| { | |
| "epoch": 2.376237623762376, | |
| "grad_norm": 0.3784586787223816, | |
| "learning_rate": 5.108818015890785e-07, | |
| "logits/chosen": -0.3249315917491913, | |
| "logits/rejected": -0.30507951974868774, | |
| "logps/chosen": -0.8853880167007446, | |
| "logps/rejected": -1.0341455936431885, | |
| "loss": 0.9531, | |
| "odds_ratio_loss": 0.6767874956130981, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.08853879570960999, | |
| "rewards/margins": 0.014875771477818489, | |
| "rewards/rejected": -0.10341457277536392, | |
| "sft_loss": 0.8853880167007446, | |
| "step": 1470 | |
| }, | |
| { | |
| "epoch": 2.392402505556678, | |
| "grad_norm": 0.5850736498832703, | |
| "learning_rate": 4.855058069687291e-07, | |
| "logits/chosen": -0.4515988230705261, | |
| "logits/rejected": -0.4501380920410156, | |
| "logps/chosen": -0.8440315127372742, | |
| "logps/rejected": -0.978651225566864, | |
| "loss": 0.9111, | |
| "odds_ratio_loss": 0.6708062887191772, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.0844031572341919, | |
| "rewards/margins": 0.013461967930197716, | |
| "rewards/rejected": -0.09786512702703476, | |
| "sft_loss": 0.8440315127372742, | |
| "step": 1480 | |
| }, | |
| { | |
| "epoch": 2.40856738735098, | |
| "grad_norm": 0.4087739884853363, | |
| "learning_rate": 4.607082849092523e-07, | |
| "logits/chosen": -0.3892877697944641, | |
| "logits/rejected": -0.4075300097465515, | |
| "logps/chosen": -0.9417757987976074, | |
| "logps/rejected": -1.0281052589416504, | |
| "loss": 1.0107, | |
| "odds_ratio_loss": 0.6892626881599426, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.09417758882045746, | |
| "rewards/margins": 0.00863293744623661, | |
| "rewards/rejected": -0.10281052440404892, | |
| "sft_loss": 0.9417757987976074, | |
| "step": 1490 | |
| }, | |
| { | |
| "epoch": 2.4247322691452817, | |
| "grad_norm": 1.142304539680481, | |
| "learning_rate": 4.3649635614901405e-07, | |
| "logits/chosen": -0.39748096466064453, | |
| "logits/rejected": -0.2932053208351135, | |
| "logps/chosen": -0.8502078056335449, | |
| "logps/rejected": -0.883902907371521, | |
| "loss": 0.9219, | |
| "odds_ratio_loss": 0.7170731425285339, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.08502078056335449, | |
| "rewards/margins": 0.003369513200595975, | |
| "rewards/rejected": -0.0883902907371521, | |
| "sft_loss": 0.8502078056335449, | |
| "step": 1500 | |
| }, | |
| { | |
| "epoch": 2.4247322691452817, | |
| "eval_logits/chosen": -0.3775150775909424, | |
| "eval_logits/rejected": -0.3581116795539856, | |
| "eval_logps/chosen": -0.8786855936050415, | |
| "eval_logps/rejected": -0.9948004484176636, | |
| "eval_loss": 0.9497246742248535, | |
| "eval_odds_ratio_loss": 0.7103896737098694, | |
| "eval_rewards/accuracies": 0.5163636207580566, | |
| "eval_rewards/chosen": -0.0878685712814331, | |
| "eval_rewards/margins": 0.011611479334533215, | |
| "eval_rewards/rejected": -0.0994800478219986, | |
| "eval_runtime": 192.2752, | |
| "eval_samples_per_second": 5.721, | |
| "eval_sft_loss": 0.8786855936050415, | |
| "eval_steps_per_second": 2.86, | |
| "step": 1500 | |
| }, | |
| { | |
| "epoch": 2.4408971509395836, | |
| "grad_norm": 0.3841034770011902, | |
| "learning_rate": 4.128769732701973e-07, | |
| "logits/chosen": -0.36835092306137085, | |
| "logits/rejected": -0.4074084758758545, | |
| "logps/chosen": -0.8371820449829102, | |
| "logps/rejected": -0.9595246315002441, | |
| "loss": 0.9062, | |
| "odds_ratio_loss": 0.6903966665267944, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.0837181955575943, | |
| "rewards/margins": 0.012234264984726906, | |
| "rewards/rejected": -0.09595246613025665, | |
| "sft_loss": 0.8371820449829102, | |
| "step": 1510 | |
| }, | |
| { | |
| "epoch": 2.4570620327338855, | |
| "grad_norm": 0.6487218737602234, | |
| "learning_rate": 3.8985691870233046e-07, | |
| "logits/chosen": -0.36084288358688354, | |
| "logits/rejected": -0.35909101366996765, | |
| "logps/chosen": -0.8767590522766113, | |
| "logps/rejected": -0.9904271364212036, | |
| "loss": 0.9487, | |
| "odds_ratio_loss": 0.7190364599227905, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.08767590671777725, | |
| "rewards/margins": 0.011366801336407661, | |
| "rewards/rejected": -0.09904270619153976, | |
| "sft_loss": 0.8767590522766113, | |
| "step": 1520 | |
| }, | |
| { | |
| "epoch": 2.4732269145281873, | |
| "grad_norm": 0.726983904838562, | |
| "learning_rate": 3.6744280277467904e-07, | |
| "logits/chosen": -0.3547779619693756, | |
| "logits/rejected": -0.37871819734573364, | |
| "logps/chosen": -0.8915858268737793, | |
| "logps/rejected": -1.0086140632629395, | |
| "loss": 0.9661, | |
| "odds_ratio_loss": 0.7449706792831421, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.08915858715772629, | |
| "rewards/margins": 0.01170281507074833, | |
| "rewards/rejected": -0.10086140781641006, | |
| "sft_loss": 0.8915858268737793, | |
| "step": 1530 | |
| }, | |
| { | |
| "epoch": 2.489391796322489, | |
| "grad_norm": 0.6208191514015198, | |
| "learning_rate": 3.456410618180503e-07, | |
| "logits/chosen": -0.46183329820632935, | |
| "logits/rejected": -0.3973988890647888, | |
| "logps/chosen": -0.7950559258460999, | |
| "logps/rejected": -1.0139881372451782, | |
| "loss": 0.8596, | |
| "odds_ratio_loss": 0.6458045244216919, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.07950559258460999, | |
| "rewards/margins": 0.021893223747611046, | |
| "rewards/rejected": -0.10139881074428558, | |
| "sft_loss": 0.7950559258460999, | |
| "step": 1540 | |
| }, | |
| { | |
| "epoch": 2.5055566781167915, | |
| "grad_norm": 0.40934354066848755, | |
| "learning_rate": 3.244579563165753e-07, | |
| "logits/chosen": -0.3711478114128113, | |
| "logits/rejected": -0.3300473093986511, | |
| "logps/chosen": -0.8490577936172485, | |
| "logps/rejected": -1.0569615364074707, | |
| "loss": 0.9137, | |
| "odds_ratio_loss": 0.6463108062744141, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.08490578085184097, | |
| "rewards/margins": 0.020790381357073784, | |
| "rewards/rejected": -0.10569615662097931, | |
| "sft_loss": 0.8490577936172485, | |
| "step": 1550 | |
| }, | |
| { | |
| "epoch": 2.521721559911093, | |
| "grad_norm": 0.4264324903488159, | |
| "learning_rate": 3.038995691099697e-07, | |
| "logits/chosen": -0.35405951738357544, | |
| "logits/rejected": -0.3723445534706116, | |
| "logps/chosen": -0.8575676083564758, | |
| "logps/rejected": -1.0358964204788208, | |
| "loss": 0.9267, | |
| "odds_ratio_loss": 0.6915205717086792, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.08575676381587982, | |
| "rewards/margins": 0.017832884564995766, | |
| "rewards/rejected": -0.10358965396881104, | |
| "sft_loss": 0.8575676083564758, | |
| "step": 1560 | |
| }, | |
| { | |
| "epoch": 2.5378864417053952, | |
| "grad_norm": 0.5124202370643616, | |
| "learning_rate": 2.839718036468192e-07, | |
| "logits/chosen": -0.39767321944236755, | |
| "logits/rejected": -0.361719012260437, | |
| "logps/chosen": -0.9866407513618469, | |
| "logps/rejected": -1.0687347650527954, | |
| "loss": 1.0574, | |
| "odds_ratio_loss": 0.7079859972000122, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.09866407513618469, | |
| "rewards/margins": 0.00820938404649496, | |
| "rewards/rejected": -0.10687346756458282, | |
| "sft_loss": 0.9866407513618469, | |
| "step": 1570 | |
| }, | |
| { | |
| "epoch": 2.5540513234996967, | |
| "grad_norm": 0.6700158715248108, | |
| "learning_rate": 2.646803822893723e-07, | |
| "logits/chosen": -0.34473222494125366, | |
| "logits/rejected": -0.339333713054657, | |
| "logps/chosen": -0.9860366582870483, | |
| "logps/rejected": -1.0728685855865479, | |
| "loss": 1.0579, | |
| "odds_ratio_loss": 0.7182521224021912, | |
| "rewards/accuracies": 0.4749999940395355, | |
| "rewards/chosen": -0.09860367327928543, | |
| "rewards/margins": 0.008683168329298496, | |
| "rewards/rejected": -0.10728684812784195, | |
| "sft_loss": 0.9860366582870483, | |
| "step": 1580 | |
| }, | |
| { | |
| "epoch": 2.570216205293999, | |
| "grad_norm": 0.4476275146007538, | |
| "learning_rate": 2.460308446703341e-07, | |
| "logits/chosen": -0.37150639295578003, | |
| "logits/rejected": -0.3977029621601105, | |
| "logps/chosen": -0.8994391560554504, | |
| "logps/rejected": -0.9403126835823059, | |
| "loss": 0.9704, | |
| "odds_ratio_loss": 0.7100769877433777, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.08994391560554504, | |
| "rewards/margins": 0.004087349865585566, | |
| "rewards/rejected": -0.09403126686811447, | |
| "sft_loss": 0.8994391560554504, | |
| "step": 1590 | |
| }, | |
| { | |
| "epoch": 2.5863810870883004, | |
| "grad_norm": 0.8473093509674072, | |
| "learning_rate": 2.2802854610213143e-07, | |
| "logits/chosen": -0.38676199316978455, | |
| "logits/rejected": -0.3973104655742645, | |
| "logps/chosen": -0.8438700437545776, | |
| "logps/rejected": -1.018701434135437, | |
| "loss": 0.9107, | |
| "odds_ratio_loss": 0.6678277850151062, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.08438700437545776, | |
| "rewards/margins": 0.017483150586485863, | |
| "rewards/rejected": -0.10187015682458878, | |
| "sft_loss": 0.8438700437545776, | |
| "step": 1600 | |
| }, | |
| { | |
| "epoch": 2.6025459688826027, | |
| "grad_norm": 1.2318559885025024, | |
| "learning_rate": 2.106786560391072e-07, | |
| "logits/chosen": -0.41062861680984497, | |
| "logits/rejected": -0.3663537800312042, | |
| "logps/chosen": -0.9180322885513306, | |
| "logps/rejected": -0.9797943830490112, | |
| "loss": 0.9881, | |
| "odds_ratio_loss": 0.7011545300483704, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.09180323779582977, | |
| "rewards/margins": 0.006176213268190622, | |
| "rewards/rejected": -0.09797944128513336, | |
| "sft_loss": 0.9180322885513306, | |
| "step": 1610 | |
| }, | |
| { | |
| "epoch": 2.6187108506769046, | |
| "grad_norm": 1.8344284296035767, | |
| "learning_rate": 1.9398615659308255e-07, | |
| "logits/chosen": -0.3516565263271332, | |
| "logits/rejected": -0.3090236485004425, | |
| "logps/chosen": -0.8868433833122253, | |
| "logps/rejected": -0.9610105752944946, | |
| "loss": 0.9563, | |
| "odds_ratio_loss": 0.6944981813430786, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.08868434280157089, | |
| "rewards/margins": 0.007416720036417246, | |
| "rewards/rejected": -0.0961010605096817, | |
| "sft_loss": 0.8868433833122253, | |
| "step": 1620 | |
| }, | |
| { | |
| "epoch": 2.6348757324712064, | |
| "grad_norm": 3.686185359954834, | |
| "learning_rate": 1.7795584110272184e-07, | |
| "logits/chosen": -0.33260416984558105, | |
| "logits/rejected": -0.32040587067604065, | |
| "logps/chosen": -0.9077906608581543, | |
| "logps/rejected": -1.0257583856582642, | |
| "loss": 0.9756, | |
| "odds_ratio_loss": 0.6781536340713501, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.09077905863523483, | |
| "rewards/margins": 0.01179676502943039, | |
| "rewards/rejected": -0.10257583856582642, | |
| "sft_loss": 0.9077906608581543, | |
| "step": 1630 | |
| }, | |
| { | |
| "epoch": 2.6510406142655083, | |
| "grad_norm": 0.7552462220191956, | |
| "learning_rate": 1.6259231275709636e-07, | |
| "logits/chosen": -0.32405030727386475, | |
| "logits/rejected": -0.3262009024620056, | |
| "logps/chosen": -0.8568581342697144, | |
| "logps/rejected": -0.9373190999031067, | |
| "loss": 0.9294, | |
| "odds_ratio_loss": 0.7254046201705933, | |
| "rewards/accuracies": 0.4749999940395355, | |
| "rewards/chosen": -0.08568581938743591, | |
| "rewards/margins": 0.008046089671552181, | |
| "rewards/rejected": -0.09373190253973007, | |
| "sft_loss": 0.8568581342697144, | |
| "step": 1640 | |
| }, | |
| { | |
| "epoch": 2.66720549605981, | |
| "grad_norm": 0.45023104548454285, | |
| "learning_rate": 1.478999832738548e-07, | |
| "logits/chosen": -0.34250158071517944, | |
| "logits/rejected": -0.34709858894348145, | |
| "logps/chosen": -0.8354190587997437, | |
| "logps/rejected": -0.9979323148727417, | |
| "loss": 0.9021, | |
| "odds_ratio_loss": 0.6672018766403198, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.08354191482067108, | |
| "rewards/margins": 0.01625131070613861, | |
| "rewards/rejected": -0.09979323297739029, | |
| "sft_loss": 0.8354190587997437, | |
| "step": 1650 | |
| }, | |
| { | |
| "epoch": 2.683370377854112, | |
| "grad_norm": 0.6760185956954956, | |
| "learning_rate": 1.338830716323769e-07, | |
| "logits/chosen": -0.34901902079582214, | |
| "logits/rejected": -0.352342426776886, | |
| "logps/chosen": -0.8232784271240234, | |
| "logps/rejected": -0.9058715105056763, | |
| "loss": 0.8916, | |
| "odds_ratio_loss": 0.6835728883743286, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.08232785016298294, | |
| "rewards/margins": 0.008259310387074947, | |
| "rewards/rejected": -0.09058715403079987, | |
| "sft_loss": 0.8232784271240234, | |
| "step": 1660 | |
| }, | |
| { | |
| "epoch": 2.699535259648414, | |
| "grad_norm": 0.9901576638221741, | |
| "learning_rate": 1.205456028622723e-07, | |
| "logits/chosen": -0.3495160639286041, | |
| "logits/rejected": -0.35691842436790466, | |
| "logps/chosen": -0.8500292897224426, | |
| "logps/rejected": -1.0147500038146973, | |
| "loss": 0.9171, | |
| "odds_ratio_loss": 0.6710700988769531, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.08500292897224426, | |
| "rewards/margins": 0.016472063958644867, | |
| "rewards/rejected": -0.10147500038146973, | |
| "sft_loss": 0.8500292897224426, | |
| "step": 1670 | |
| }, | |
| { | |
| "epoch": 2.7157001414427158, | |
| "grad_norm": 0.29376673698425293, | |
| "learning_rate": 1.0789140688756805e-07, | |
| "logits/chosen": -0.2777409255504608, | |
| "logits/rejected": -0.30515843629837036, | |
| "logps/chosen": -0.8388081789016724, | |
| "logps/rejected": -1.004902720451355, | |
| "loss": 0.9016, | |
| "odds_ratio_loss": 0.6277891397476196, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.08388081192970276, | |
| "rewards/margins": 0.016609463840723038, | |
| "rewards/rejected": -0.10049028694629669, | |
| "sft_loss": 0.8388081789016724, | |
| "step": 1680 | |
| }, | |
| { | |
| "epoch": 2.7318650232370176, | |
| "grad_norm": 1.1649651527404785, | |
| "learning_rate": 9.592411742693098e-08, | |
| "logits/chosen": -0.3376592993736267, | |
| "logits/rejected": -0.33899828791618347, | |
| "logps/chosen": -0.8941831588745117, | |
| "logps/rejected": -0.9593558311462402, | |
| "loss": 0.9688, | |
| "odds_ratio_loss": 0.7464134693145752, | |
| "rewards/accuracies": 0.44999998807907104, | |
| "rewards/chosen": -0.08941832929849625, | |
| "rewards/margins": 0.00651725847274065, | |
| "rewards/rejected": -0.09593559056520462, | |
| "sft_loss": 0.8941831588745117, | |
| "step": 1690 | |
| }, | |
| { | |
| "epoch": 2.7480299050313195, | |
| "grad_norm": 0.365510493516922, | |
| "learning_rate": 8.464717095022168e-08, | |
| "logits/chosen": -0.26350411772727966, | |
| "logits/rejected": -0.3258097767829895, | |
| "logps/chosen": -0.8289276957511902, | |
| "logps/rejected": -0.9933468103408813, | |
| "loss": 0.894, | |
| "odds_ratio_loss": 0.6506984829902649, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.0828927755355835, | |
| "rewards/margins": 0.01644190214574337, | |
| "rewards/rejected": -0.09933467954397202, | |
| "sft_loss": 0.8289276957511902, | |
| "step": 1700 | |
| }, | |
| { | |
| "epoch": 2.7641947868256214, | |
| "grad_norm": 0.860230565071106, | |
| "learning_rate": 7.406380569169841e-08, | |
| "logits/chosen": -0.35509008169174194, | |
| "logits/rejected": -0.3218967318534851, | |
| "logps/chosen": -0.9126371145248413, | |
| "logps/rejected": -0.8999163508415222, | |
| "loss": 0.9886, | |
| "odds_ratio_loss": 0.759522020816803, | |
| "rewards/accuracies": 0.4437499940395355, | |
| "rewards/chosen": -0.09126370400190353, | |
| "rewards/margins": -0.0012720691738650203, | |
| "rewards/rejected": -0.08999162912368774, | |
| "sft_loss": 0.9126371145248413, | |
| "step": 1710 | |
| }, | |
| { | |
| "epoch": 2.7803596686199232, | |
| "grad_norm": 2.069009780883789, | |
| "learning_rate": 6.417706072013808e-08, | |
| "logits/chosen": -0.3513588011264801, | |
| "logits/rejected": -0.31902140378952026, | |
| "logps/chosen": -0.8999738693237305, | |
| "logps/rejected": -0.9839135408401489, | |
| "loss": 0.9715, | |
| "odds_ratio_loss": 0.7152166366577148, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.08999738842248917, | |
| "rewards/margins": 0.008393971249461174, | |
| "rewards/rejected": -0.09839136153459549, | |
| "sft_loss": 0.8999738693237305, | |
| "step": 1720 | |
| }, | |
| { | |
| "epoch": 2.796524550414225, | |
| "grad_norm": 0.59537672996521, | |
| "learning_rate": 5.498977506615294e-08, | |
| "logits/chosen": -0.33539581298828125, | |
| "logits/rejected": -0.36086633801460266, | |
| "logps/chosen": -0.8895516395568848, | |
| "logps/rejected": -0.9674522280693054, | |
| "loss": 0.9602, | |
| "odds_ratio_loss": 0.706065833568573, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.08895515650510788, | |
| "rewards/margins": 0.007790066301822662, | |
| "rewards/rejected": -0.09674523025751114, | |
| "sft_loss": 0.8895516395568848, | |
| "step": 1730 | |
| }, | |
| { | |
| "epoch": 2.812689432208527, | |
| "grad_norm": 0.4070757031440735, | |
| "learning_rate": 4.6504586906947756e-08, | |
| "logits/chosen": -0.3671857714653015, | |
| "logits/rejected": -0.36166203022003174, | |
| "logps/chosen": -0.9486915469169617, | |
| "logps/rejected": -0.9999829530715942, | |
| "loss": 1.0182, | |
| "odds_ratio_loss": 0.6954110860824585, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.09486915171146393, | |
| "rewards/margins": 0.005129144061356783, | |
| "rewards/rejected": -0.09999830275774002, | |
| "sft_loss": 0.9486915469169617, | |
| "step": 1740 | |
| }, | |
| { | |
| "epoch": 2.828854314002829, | |
| "grad_norm": 1.650687336921692, | |
| "learning_rate": 3.8723932808754914e-08, | |
| "logits/chosen": -0.2851547300815582, | |
| "logits/rejected": -0.2857135236263275, | |
| "logps/chosen": -0.9708272814750671, | |
| "logps/rejected": -0.9912136197090149, | |
| "loss": 1.0459, | |
| "odds_ratio_loss": 0.7506999969482422, | |
| "rewards/accuracies": 0.46875, | |
| "rewards/chosen": -0.09708271920681, | |
| "rewards/margins": 0.0020386301912367344, | |
| "rewards/rejected": -0.0991213470697403, | |
| "sft_loss": 0.9708272814750671, | |
| "step": 1750 | |
| }, | |
| { | |
| "epoch": 2.8450191957971307, | |
| "grad_norm": 0.9035086035728455, | |
| "learning_rate": 3.1650047027158014e-08, | |
| "logits/chosen": -0.3378879427909851, | |
| "logits/rejected": -0.31768563389778137, | |
| "logps/chosen": -0.863334059715271, | |
| "logps/rejected": -0.9955730438232422, | |
| "loss": 0.9285, | |
| "odds_ratio_loss": 0.6513949632644653, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.08633340895175934, | |
| "rewards/margins": 0.013223896734416485, | |
| "rewards/rejected": -0.0995573028922081, | |
| "sft_loss": 0.863334059715271, | |
| "step": 1760 | |
| }, | |
| { | |
| "epoch": 2.8611840775914326, | |
| "grad_norm": 0.3864952623844147, | |
| "learning_rate": 2.5284960865517848e-08, | |
| "logits/chosen": -0.39154380559921265, | |
| "logits/rejected": -0.34484562277793884, | |
| "logps/chosen": -0.82793790102005, | |
| "logps/rejected": -1.0070700645446777, | |
| "loss": 0.8928, | |
| "odds_ratio_loss": 0.6486603021621704, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.08279379457235336, | |
| "rewards/margins": 0.0179132129997015, | |
| "rewards/rejected": -0.10070700943470001, | |
| "sft_loss": 0.82793790102005, | |
| "step": 1770 | |
| }, | |
| { | |
| "epoch": 2.8773489593857344, | |
| "grad_norm": 0.4017253518104553, | |
| "learning_rate": 1.9630502091670388e-08, | |
| "logits/chosen": -0.3473368287086487, | |
| "logits/rejected": -0.37853848934173584, | |
| "logps/chosen": -0.8299247622489929, | |
| "logps/rejected": -0.996843695640564, | |
| "loss": 0.8926, | |
| "odds_ratio_loss": 0.6264339685440063, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.08299248665571213, | |
| "rewards/margins": 0.016691887751221657, | |
| "rewards/rejected": -0.09968437254428864, | |
| "sft_loss": 0.8299247622489929, | |
| "step": 1780 | |
| }, | |
| { | |
| "epoch": 2.8935138411800363, | |
| "grad_norm": 0.7657872438430786, | |
| "learning_rate": 1.4688294413074677e-08, | |
| "logits/chosen": -0.3813559114933014, | |
| "logits/rejected": -0.34783899784088135, | |
| "logps/chosen": -0.802249550819397, | |
| "logps/rejected": -0.9486366510391235, | |
| "loss": 0.8723, | |
| "odds_ratio_loss": 0.7008516788482666, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.08022496104240417, | |
| "rewards/margins": 0.014638709835708141, | |
| "rewards/rejected": -0.09486366808414459, | |
| "sft_loss": 0.802249550819397, | |
| "step": 1790 | |
| }, | |
| { | |
| "epoch": 2.909678722974338, | |
| "grad_norm": 0.2962876558303833, | |
| "learning_rate": 1.0459757010556626e-08, | |
| "logits/chosen": -0.4134625494480133, | |
| "logits/rejected": -0.394450843334198, | |
| "logps/chosen": -0.8447575569152832, | |
| "logps/rejected": -0.919145405292511, | |
| "loss": 0.9156, | |
| "odds_ratio_loss": 0.7088185548782349, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.08447576314210892, | |
| "rewards/margins": 0.007438770029693842, | |
| "rewards/rejected": -0.09191453456878662, | |
| "sft_loss": 0.8447575569152832, | |
| "step": 1800 | |
| }, | |
| { | |
| "epoch": 2.92584360476864, | |
| "grad_norm": 0.4512230455875397, | |
| "learning_rate": 6.94610413078306e-09, | |
| "logits/chosen": -0.4446278512477875, | |
| "logits/rejected": -0.37901362776756287, | |
| "logps/chosen": -0.8928766250610352, | |
| "logps/rejected": -1.078958511352539, | |
| "loss": 0.9617, | |
| "odds_ratio_loss": 0.6879509091377258, | |
| "rewards/accuracies": 0.48124998807907104, | |
| "rewards/chosen": -0.0892876610159874, | |
| "rewards/margins": 0.01860819011926651, | |
| "rewards/rejected": -0.1078958511352539, | |
| "sft_loss": 0.8928766250610352, | |
| "step": 1810 | |
| }, | |
| { | |
| "epoch": 2.942008486562942, | |
| "grad_norm": 0.2540852427482605, | |
| "learning_rate": 4.14834473758563e-09, | |
| "logits/chosen": -0.4007115364074707, | |
| "logits/rejected": -0.3911517858505249, | |
| "logps/chosen": -0.8001864552497864, | |
| "logps/rejected": -1.0187556743621826, | |
| "loss": 0.8634, | |
| "odds_ratio_loss": 0.6319615244865417, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.08001864701509476, | |
| "rewards/margins": 0.021856937557458878, | |
| "rewards/rejected": -0.10187558084726334, | |
| "sft_loss": 0.8001864552497864, | |
| "step": 1820 | |
| }, | |
| { | |
| "epoch": 2.9581733683572438, | |
| "grad_norm": 0.4121166467666626, | |
| "learning_rate": 2.067282222230349e-09, | |
| "logits/chosen": -0.3413907587528229, | |
| "logits/rejected": -0.278145968914032, | |
| "logps/chosen": -0.8189884424209595, | |
| "logps/rejected": -1.0053989887237549, | |
| "loss": 0.881, | |
| "odds_ratio_loss": 0.620233416557312, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.08189885318279266, | |
| "rewards/margins": 0.018641049042344093, | |
| "rewards/rejected": -0.10053990036249161, | |
| "sft_loss": 0.8189884424209595, | |
| "step": 1830 | |
| }, | |
| { | |
| "epoch": 2.9743382501515456, | |
| "grad_norm": 3.4636123180389404, | |
| "learning_rate": 7.035141727212979e-10, | |
| "logits/chosen": -0.3847911059856415, | |
| "logits/rejected": -0.34176406264305115, | |
| "logps/chosen": -0.8342105746269226, | |
| "logps/rejected": -0.9381749033927917, | |
| "loss": 0.9018, | |
| "odds_ratio_loss": 0.6754266023635864, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.08342105895280838, | |
| "rewards/margins": 0.010396432131528854, | |
| "rewards/rejected": -0.09381748735904694, | |
| "sft_loss": 0.8342105746269226, | |
| "step": 1840 | |
| }, | |
| { | |
| "epoch": 2.9905031319458475, | |
| "grad_norm": 1.2374101877212524, | |
| "learning_rate": 5.743220219761592e-11, | |
| "logits/chosen": -0.33420827984809875, | |
| "logits/rejected": -0.3142699599266052, | |
| "logps/chosen": -1.0187790393829346, | |
| "logps/rejected": -1.029541015625, | |
| "loss": 1.097, | |
| "odds_ratio_loss": 0.782578706741333, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.10187790542840958, | |
| "rewards/margins": 0.0010761909652501345, | |
| "rewards/rejected": -0.10295410454273224, | |
| "sft_loss": 1.0187790393829346, | |
| "step": 1850 | |
| }, | |
| { | |
| "epoch": 2.9969690846635686, | |
| "step": 1854, | |
| "total_flos": 1.9948570754930442e+18, | |
| "train_loss": 0.9750770799807618, | |
| "train_runtime": 17949.5667, | |
| "train_samples_per_second": 1.654, | |
| "train_steps_per_second": 0.103 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 1854, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 500, | |
| "total_flos": 1.9948570754930442e+18, | |
| "train_batch_size": 2, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |