| { | |
| "best_global_step": null, | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 2.610526315789474, | |
| "eval_steps": 500, | |
| "global_step": 125, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.021052631578947368, | |
| "grad_norm": 1.224082589149475, | |
| "kl": 0.0, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": 270973525.3333333, | |
| "logits/rejected": 308584228.5714286, | |
| "logps/chosen": -743.7762586805555, | |
| "logps/rejected": -327.42550223214283, | |
| "loss": 0.5, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.042105263157894736, | |
| "grad_norm": 1.7011622190475464, | |
| "kl": 0.08848989009857178, | |
| "learning_rate": 4.999405067699773e-06, | |
| "logits/chosen": 283945024.0, | |
| "logits/rejected": 300706848.0, | |
| "logps/chosen": -670.29150390625, | |
| "logps/rejected": -340.8790283203125, | |
| "loss": 0.5072, | |
| "rewards/chosen": -0.04161600396037102, | |
| "rewards/margins": -0.051780181005597115, | |
| "rewards/rejected": 0.010164177045226097, | |
| "step": 2 | |
| }, | |
| { | |
| "epoch": 0.06315789473684211, | |
| "grad_norm": 1.2693746089935303, | |
| "kl": 0.17015418410301208, | |
| "learning_rate": 4.997620553954645e-06, | |
| "logits/chosen": 268529444.5714286, | |
| "logits/rejected": 305261397.3333333, | |
| "logps/chosen": -813.0807756696429, | |
| "logps/rejected": -319.21929253472223, | |
| "loss": 0.496, | |
| "rewards/chosen": 0.018497141344206675, | |
| "rewards/margins": 0.028634450974918547, | |
| "rewards/rejected": -0.010137309630711874, | |
| "step": 3 | |
| }, | |
| { | |
| "epoch": 0.08421052631578947, | |
| "grad_norm": 2.279048442840576, | |
| "kl": 0.06164896488189697, | |
| "learning_rate": 4.994647308096509e-06, | |
| "logits/chosen": 252110563.55555555, | |
| "logits/rejected": 306722450.28571427, | |
| "logps/chosen": -748.6314019097222, | |
| "logps/rejected": -325.29725864955356, | |
| "loss": 0.4976, | |
| "rewards/chosen": 0.011831367181407081, | |
| "rewards/margins": 0.011405953797437841, | |
| "rewards/rejected": 0.0004254133839692388, | |
| "step": 4 | |
| }, | |
| { | |
| "epoch": 0.10526315789473684, | |
| "grad_norm": 1.5578685998916626, | |
| "kl": 0.06295323371887207, | |
| "learning_rate": 4.990486745229364e-06, | |
| "logits/chosen": 292113728.0, | |
| "logits/rejected": 316883904.0, | |
| "logps/chosen": -815.0404663085938, | |
| "logps/rejected": -337.57080078125, | |
| "loss": 0.499, | |
| "rewards/chosen": 0.006664060987532139, | |
| "rewards/margins": 0.011198383755981922, | |
| "rewards/rejected": -0.004534322768449783, | |
| "step": 5 | |
| }, | |
| { | |
| "epoch": 0.12631578947368421, | |
| "grad_norm": 1.006475567817688, | |
| "kl": 0.049591064453125, | |
| "learning_rate": 4.985140845555799e-06, | |
| "logits/chosen": 306853741.71428573, | |
| "logits/rejected": 325227320.8888889, | |
| "logps/chosen": -479.3779994419643, | |
| "logps/rejected": -286.5568576388889, | |
| "loss": 0.4958, | |
| "rewards/chosen": 0.026198712842805044, | |
| "rewards/margins": 0.030564389649837737, | |
| "rewards/rejected": -0.004365676807032691, | |
| "step": 6 | |
| }, | |
| { | |
| "epoch": 0.14736842105263157, | |
| "grad_norm": 1.2592947483062744, | |
| "kl": 0.15429818630218506, | |
| "learning_rate": 4.978612153434527e-06, | |
| "logits/chosen": 305409763.5555556, | |
| "logits/rejected": 309714139.4285714, | |
| "logps/chosen": -609.1155056423611, | |
| "logps/rejected": -277.2419956752232, | |
| "loss": 0.4977, | |
| "rewards/chosen": 0.01816416945722368, | |
| "rewards/margins": 0.028014377705634586, | |
| "rewards/rejected": -0.009850208248410906, | |
| "step": 7 | |
| }, | |
| { | |
| "epoch": 0.16842105263157894, | |
| "grad_norm": 1.420854926109314, | |
| "kl": 0.20131784677505493, | |
| "learning_rate": 4.970903776169403e-06, | |
| "logits/chosen": 300502112.0, | |
| "logits/rejected": 337999712.0, | |
| "logps/chosen": -599.28125, | |
| "logps/rejected": -321.2285461425781, | |
| "loss": 0.5003, | |
| "rewards/chosen": 0.006110990885645151, | |
| "rewards/margins": 0.016330440063029528, | |
| "rewards/rejected": -0.010219449177384377, | |
| "step": 8 | |
| }, | |
| { | |
| "epoch": 0.18947368421052632, | |
| "grad_norm": 1.0294359922409058, | |
| "kl": 0.15416035056114197, | |
| "learning_rate": 4.962019382530521e-06, | |
| "logits/chosen": 280579858.28571427, | |
| "logits/rejected": 310000554.6666667, | |
| "logps/chosen": -656.5274135044643, | |
| "logps/rejected": -314.7470703125, | |
| "loss": 0.4943, | |
| "rewards/chosen": 0.05247082880565098, | |
| "rewards/margins": 0.061781181111222216, | |
| "rewards/rejected": -0.009310352305571238, | |
| "step": 9 | |
| }, | |
| { | |
| "epoch": 0.21052631578947367, | |
| "grad_norm": 1.8129311800003052, | |
| "kl": 0.1927608847618103, | |
| "learning_rate": 4.9519632010080765e-06, | |
| "logits/chosen": 238834005.33333334, | |
| "logits/rejected": 301916818.28571427, | |
| "logps/chosen": -759.1360134548611, | |
| "logps/rejected": -292.9808872767857, | |
| "loss": 0.5004, | |
| "rewards/chosen": 0.014881134033203125, | |
| "rewards/margins": 0.019827809184789658, | |
| "rewards/rejected": -0.004946675151586533, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.23157894736842105, | |
| "grad_norm": 1.232249140739441, | |
| "kl": 0.23234114050865173, | |
| "learning_rate": 4.9407400177998335e-06, | |
| "logits/chosen": 286699296.0, | |
| "logits/rejected": 300436576.0, | |
| "logps/chosen": -627.421875, | |
| "logps/rejected": -282.529296875, | |
| "loss": 0.4981, | |
| "rewards/chosen": 0.03665875270962715, | |
| "rewards/margins": 0.0362735278904438, | |
| "rewards/rejected": 0.0003852248191833496, | |
| "step": 11 | |
| }, | |
| { | |
| "epoch": 0.25263157894736843, | |
| "grad_norm": 1.2517226934432983, | |
| "kl": 0.2269219160079956, | |
| "learning_rate": 4.928355174533153e-06, | |
| "logits/chosen": 283396937.14285713, | |
| "logits/rejected": 298269838.2222222, | |
| "logps/chosen": -700.8916015625, | |
| "logps/rejected": -346.30750868055554, | |
| "loss": 0.4953, | |
| "rewards/chosen": 0.052171528339385986, | |
| "rewards/margins": 0.05235843691560957, | |
| "rewards/rejected": -0.00018690857622358535, | |
| "step": 12 | |
| }, | |
| { | |
| "epoch": 0.2736842105263158, | |
| "grad_norm": 1.2330269813537598, | |
| "kl": 0.2519031763076782, | |
| "learning_rate": 4.914814565722671e-06, | |
| "logits/chosen": 302669795.5555556, | |
| "logits/rejected": 313485238.85714287, | |
| "logps/chosen": -654.2307942708334, | |
| "logps/rejected": -330.16427176339283, | |
| "loss": 0.4958, | |
| "rewards/chosen": 0.0363319648636712, | |
| "rewards/margins": 0.07227063652068849, | |
| "rewards/rejected": -0.0359386716570173, | |
| "step": 13 | |
| }, | |
| { | |
| "epoch": 0.29473684210526313, | |
| "grad_norm": 1.4144365787506104, | |
| "kl": 0.130226731300354, | |
| "learning_rate": 4.900124635964823e-06, | |
| "logits/chosen": 276869248.0, | |
| "logits/rejected": 289425024.0, | |
| "logps/chosen": -519.8916015625, | |
| "logps/rejected": -265.8385314941406, | |
| "loss": 0.496, | |
| "rewards/chosen": 0.03466583415865898, | |
| "rewards/margins": 0.0368356395047158, | |
| "rewards/rejected": -0.002169805346056819, | |
| "step": 14 | |
| }, | |
| { | |
| "epoch": 0.3157894736842105, | |
| "grad_norm": 1.305309772491455, | |
| "kl": 0.12101024389266968, | |
| "learning_rate": 4.884292376870567e-06, | |
| "logits/chosen": 293158875.4285714, | |
| "logits/rejected": 304345201.7777778, | |
| "logps/chosen": -500.9711216517857, | |
| "logps/rejected": -344.5473361545139, | |
| "loss": 0.4944, | |
| "rewards/chosen": 0.06058854716164725, | |
| "rewards/margins": 0.07498784883627815, | |
| "rewards/rejected": -0.014399301674630906, | |
| "step": 15 | |
| }, | |
| { | |
| "epoch": 0.3368421052631579, | |
| "grad_norm": 1.447916865348816, | |
| "kl": 0.1458943486213684, | |
| "learning_rate": 4.867325323737765e-06, | |
| "logits/chosen": 290241507.5555556, | |
| "logits/rejected": 316717494.85714287, | |
| "logps/chosen": -713.4215494791666, | |
| "logps/rejected": -295.7858189174107, | |
| "loss": 0.4897, | |
| "rewards/chosen": 0.08352628681394789, | |
| "rewards/margins": 0.09608901836096294, | |
| "rewards/rejected": -0.012562731547015054, | |
| "step": 16 | |
| }, | |
| { | |
| "epoch": 0.35789473684210527, | |
| "grad_norm": 1.3701616525650024, | |
| "kl": 0.3826329708099365, | |
| "learning_rate": 4.849231551964771e-06, | |
| "logits/chosen": 279923008.0, | |
| "logits/rejected": 295235712.0, | |
| "logps/chosen": -549.1046752929688, | |
| "logps/rejected": -335.35260009765625, | |
| "loss": 0.4927, | |
| "rewards/chosen": 0.06883127987384796, | |
| "rewards/margins": 0.08687522634863853, | |
| "rewards/rejected": -0.018043946474790573, | |
| "step": 17 | |
| }, | |
| { | |
| "epoch": 0.37894736842105264, | |
| "grad_norm": 1.467745065689087, | |
| "kl": 0.5873703956604004, | |
| "learning_rate": 4.830019673206997e-06, | |
| "logits/chosen": 278496658.28571427, | |
| "logits/rejected": 300650268.4444444, | |
| "logps/chosen": -629.6449497767857, | |
| "logps/rejected": -360.1111653645833, | |
| "loss": 0.4923, | |
| "rewards/chosen": 0.06821728178433009, | |
| "rewards/margins": 0.0920389105403234, | |
| "rewards/rejected": -0.023821628755993314, | |
| "step": 18 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "grad_norm": 1.9833446741104126, | |
| "kl": 0.7327308654785156, | |
| "learning_rate": 4.809698831278217e-06, | |
| "logits/chosen": 277803520.0, | |
| "logits/rejected": 308114651.4285714, | |
| "logps/chosen": -737.9443901909722, | |
| "logps/rejected": -338.15098353794644, | |
| "loss": 0.4907, | |
| "rewards/chosen": 0.10881086852815416, | |
| "rewards/margins": 0.15786849695538718, | |
| "rewards/rejected": -0.04905762842723301, | |
| "step": 19 | |
| }, | |
| { | |
| "epoch": 0.42105263157894735, | |
| "grad_norm": 1.154601812362671, | |
| "kl": 0.1896182894706726, | |
| "learning_rate": 4.788278697798619e-06, | |
| "logits/chosen": 283599232.0, | |
| "logits/rejected": 313346368.0, | |
| "logps/chosen": -707.3074951171875, | |
| "logps/rejected": -316.3603515625, | |
| "loss": 0.4943, | |
| "rewards/chosen": 0.04999881610274315, | |
| "rewards/margins": 0.0727224051952362, | |
| "rewards/rejected": -0.022723589092493057, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.4421052631578947, | |
| "grad_norm": 1.574962854385376, | |
| "kl": 0.4762837886810303, | |
| "learning_rate": 4.765769467591626e-06, | |
| "logits/chosen": 287853293.71428573, | |
| "logits/rejected": 302228480.0, | |
| "logps/chosen": -572.1301618303571, | |
| "logps/rejected": -282.95513237847223, | |
| "loss": 0.488, | |
| "rewards/chosen": 0.12458467483520508, | |
| "rewards/margins": 0.15258528788884482, | |
| "rewards/rejected": -0.02800061305363973, | |
| "step": 21 | |
| }, | |
| { | |
| "epoch": 0.4631578947368421, | |
| "grad_norm": 1.5850838422775269, | |
| "kl": 0.5586809515953064, | |
| "learning_rate": 4.742181853831721e-06, | |
| "logits/chosen": 290416867.5555556, | |
| "logits/rejected": 309413156.5714286, | |
| "logps/chosen": -668.2986653645834, | |
| "logps/rejected": -324.6029575892857, | |
| "loss": 0.4884, | |
| "rewards/chosen": 0.09367326895395915, | |
| "rewards/margins": 0.1388323534102667, | |
| "rewards/rejected": -0.04515908445630755, | |
| "step": 22 | |
| }, | |
| { | |
| "epoch": 0.4842105263157895, | |
| "grad_norm": 1.4497779607772827, | |
| "kl": 0.5230355262756348, | |
| "learning_rate": 4.717527082945555e-06, | |
| "logits/chosen": 280489216.0, | |
| "logits/rejected": 309412736.0, | |
| "logps/chosen": -682.4186401367188, | |
| "logps/rejected": -332.3192138671875, | |
| "loss": 0.488, | |
| "rewards/chosen": 0.1085066944360733, | |
| "rewards/margins": 0.14131877198815346, | |
| "rewards/rejected": -0.032812077552080154, | |
| "step": 23 | |
| }, | |
| { | |
| "epoch": 0.5052631578947369, | |
| "grad_norm": 1.3657130002975464, | |
| "kl": 0.5021036863327026, | |
| "learning_rate": 4.69181688926877e-06, | |
| "logits/chosen": 239562038.85714287, | |
| "logits/rejected": 313432405.3333333, | |
| "logps/chosen": -761.9135044642857, | |
| "logps/rejected": -289.5192057291667, | |
| "loss": 0.4848, | |
| "rewards/chosen": 0.1538386004311698, | |
| "rewards/margins": 0.17894491955401406, | |
| "rewards/rejected": -0.02510631912284427, | |
| "step": 24 | |
| }, | |
| { | |
| "epoch": 0.5263157894736842, | |
| "grad_norm": 1.446842074394226, | |
| "kl": 0.6509883999824524, | |
| "learning_rate": 4.665063509461098e-06, | |
| "logits/chosen": 293611036.4444444, | |
| "logits/rejected": 316255341.71428573, | |
| "logps/chosen": -721.4010416666666, | |
| "logps/rejected": -285.1228724888393, | |
| "loss": 0.4881, | |
| "rewards/chosen": 0.12438484032948811, | |
| "rewards/margins": 0.1453892659573328, | |
| "rewards/rejected": -0.021004425627844676, | |
| "step": 25 | |
| }, | |
| { | |
| "epoch": 0.5473684210526316, | |
| "grad_norm": 1.2756037712097168, | |
| "kl": 0.4022580087184906, | |
| "learning_rate": 4.637279676682367e-06, | |
| "logits/chosen": 292368704.0, | |
| "logits/rejected": 317329792.0, | |
| "logps/chosen": -469.4183044433594, | |
| "logps/rejected": -302.54144287109375, | |
| "loss": 0.4966, | |
| "rewards/chosen": 0.07243937253952026, | |
| "rewards/margins": 0.10708872973918915, | |
| "rewards/rejected": -0.034649357199668884, | |
| "step": 26 | |
| }, | |
| { | |
| "epoch": 0.5684210526315789, | |
| "grad_norm": 1.2571344375610352, | |
| "kl": 1.006415605545044, | |
| "learning_rate": 4.608478614532215e-06, | |
| "logits/chosen": 252147291.42857143, | |
| "logits/rejected": 306243811.5555556, | |
| "logps/chosen": -751.1729910714286, | |
| "logps/rejected": -298.89708116319446, | |
| "loss": 0.4868, | |
| "rewards/chosen": 0.15643044880458287, | |
| "rewards/margins": 0.2008941164092412, | |
| "rewards/rejected": -0.044463667604658336, | |
| "step": 27 | |
| }, | |
| { | |
| "epoch": 0.5894736842105263, | |
| "grad_norm": 1.2348135709762573, | |
| "kl": 0.844231903553009, | |
| "learning_rate": 4.578674030756364e-06, | |
| "logits/chosen": 328972714.6666667, | |
| "logits/rejected": 334514980.5714286, | |
| "logps/chosen": -661.2038302951389, | |
| "logps/rejected": -348.25048828125, | |
| "loss": 0.4897, | |
| "rewards/chosen": 0.1231810384326511, | |
| "rewards/margins": 0.1448404531157206, | |
| "rewards/rejected": -0.021659414683069502, | |
| "step": 28 | |
| }, | |
| { | |
| "epoch": 0.6105263157894737, | |
| "grad_norm": 1.038001298904419, | |
| "kl": 0.5513710379600525, | |
| "learning_rate": 4.54788011072248e-06, | |
| "logits/chosen": 276993920.0, | |
| "logits/rejected": 302539648.0, | |
| "logps/chosen": -632.3786010742188, | |
| "logps/rejected": -318.97100830078125, | |
| "loss": 0.489, | |
| "rewards/chosen": 0.10169073939323425, | |
| "rewards/margins": 0.1353834606707096, | |
| "rewards/rejected": -0.03369272127747536, | |
| "step": 29 | |
| }, | |
| { | |
| "epoch": 0.631578947368421, | |
| "grad_norm": 1.5136151313781738, | |
| "kl": 0.7220326066017151, | |
| "learning_rate": 4.516111510668707e-06, | |
| "logits/chosen": 276634130.28571427, | |
| "logits/rejected": 300450048.0, | |
| "logps/chosen": -667.6506696428571, | |
| "logps/rejected": -341.21375868055554, | |
| "loss": 0.4831, | |
| "rewards/chosen": 0.2004882778440203, | |
| "rewards/margins": 0.24258499533411054, | |
| "rewards/rejected": -0.042096717490090266, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.6526315789473685, | |
| "grad_norm": 1.0843844413757324, | |
| "kl": 0.6163355112075806, | |
| "learning_rate": 4.4833833507280884e-06, | |
| "logits/chosen": 307501454.2222222, | |
| "logits/rejected": 303213787.4285714, | |
| "logps/chosen": -486.1819661458333, | |
| "logps/rejected": -337.0064174107143, | |
| "loss": 0.4833, | |
| "rewards/chosen": 0.12439311875237359, | |
| "rewards/margins": 0.1898724100892506, | |
| "rewards/rejected": -0.06547929133687701, | |
| "step": 31 | |
| }, | |
| { | |
| "epoch": 0.6736842105263158, | |
| "grad_norm": 1.1725709438323975, | |
| "kl": 0.6453120708465576, | |
| "learning_rate": 4.4497112077322045e-06, | |
| "logits/chosen": 314310528.0, | |
| "logits/rejected": 323297440.0, | |
| "logps/chosen": -570.1083374023438, | |
| "logps/rejected": -277.752197265625, | |
| "loss": 0.4858, | |
| "rewards/chosen": 0.13651692867279053, | |
| "rewards/margins": 0.190122302621603, | |
| "rewards/rejected": -0.053605373948812485, | |
| "step": 32 | |
| }, | |
| { | |
| "epoch": 0.6947368421052632, | |
| "grad_norm": 1.653479814529419, | |
| "kl": 1.0490498542785645, | |
| "learning_rate": 4.415111107797445e-06, | |
| "logits/chosen": 311787922.28571427, | |
| "logits/rejected": 350929635.5555556, | |
| "logps/chosen": -676.2388392857143, | |
| "logps/rejected": -374.4116482204861, | |
| "loss": 0.4785, | |
| "rewards/chosen": 0.2800070898873465, | |
| "rewards/margins": 0.29183324911291636, | |
| "rewards/rejected": -0.011826159225569831, | |
| "step": 33 | |
| }, | |
| { | |
| "epoch": 0.7157894736842105, | |
| "grad_norm": 1.5196908712387085, | |
| "kl": 0.8070676922798157, | |
| "learning_rate": 4.379599518697444e-06, | |
| "logits/chosen": 278536049.7777778, | |
| "logits/rejected": 313191350.85714287, | |
| "logps/chosen": -602.8457573784722, | |
| "logps/rejected": -309.17257254464283, | |
| "loss": 0.475, | |
| "rewards/chosen": 0.19871669345431858, | |
| "rewards/margins": 0.28476871952177985, | |
| "rewards/rejected": -0.08605202606746129, | |
| "step": 34 | |
| }, | |
| { | |
| "epoch": 0.7368421052631579, | |
| "grad_norm": 1.0484706163406372, | |
| "kl": 0.807353138923645, | |
| "learning_rate": 4.34319334202531e-06, | |
| "logits/chosen": 299660160.0, | |
| "logits/rejected": 306065632.0, | |
| "logps/chosen": -465.7568359375, | |
| "logps/rejected": -229.71621704101562, | |
| "loss": 0.4919, | |
| "rewards/chosen": 0.11409495025873184, | |
| "rewards/margins": 0.16953209787607193, | |
| "rewards/rejected": -0.05543714761734009, | |
| "step": 35 | |
| }, | |
| { | |
| "epoch": 0.7578947368421053, | |
| "grad_norm": 1.3274704217910767, | |
| "kl": 0.8778089880943298, | |
| "learning_rate": 4.305909905149389e-06, | |
| "logits/chosen": 270111561.14285713, | |
| "logits/rejected": 306747079.1111111, | |
| "logps/chosen": -576.6547502790179, | |
| "logps/rejected": -307.5883517795139, | |
| "loss": 0.4756, | |
| "rewards/chosen": 0.23983100482395717, | |
| "rewards/margins": 0.28282778887521653, | |
| "rewards/rejected": -0.04299678405125936, | |
| "step": 36 | |
| }, | |
| { | |
| "epoch": 0.7789473684210526, | |
| "grad_norm": 1.6128290891647339, | |
| "kl": 1.20198392868042, | |
| "learning_rate": 4.267766952966369e-06, | |
| "logits/chosen": 279583146.6666667, | |
| "logits/rejected": 321376694.85714287, | |
| "logps/chosen": -716.5959201388889, | |
| "logps/rejected": -284.7782505580357, | |
| "loss": 0.4762, | |
| "rewards/chosen": 0.2411472267574734, | |
| "rewards/margins": 0.34859214321015375, | |
| "rewards/rejected": -0.10744491645268031, | |
| "step": 37 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "grad_norm": 1.4675477743148804, | |
| "kl": 0.7991436719894409, | |
| "learning_rate": 4.228782639455674e-06, | |
| "logits/chosen": 289123392.0, | |
| "logits/rejected": 312878304.0, | |
| "logps/chosen": -548.8831176757812, | |
| "logps/rejected": -298.0077819824219, | |
| "loss": 0.4854, | |
| "rewards/chosen": 0.1557559221982956, | |
| "rewards/margins": 0.20764141902327538, | |
| "rewards/rejected": -0.05188549682497978, | |
| "step": 38 | |
| }, | |
| { | |
| "epoch": 0.8210526315789474, | |
| "grad_norm": 1.5274215936660767, | |
| "kl": 1.3201950788497925, | |
| "learning_rate": 4.188975519039151e-06, | |
| "logits/chosen": 306054198.85714287, | |
| "logits/rejected": 318293873.7777778, | |
| "logps/chosen": -641.423828125, | |
| "logps/rejected": -323.07017686631946, | |
| "loss": 0.4675, | |
| "rewards/chosen": 0.2957319532121931, | |
| "rewards/margins": 0.35744103268971517, | |
| "rewards/rejected": -0.061709079477522105, | |
| "step": 39 | |
| }, | |
| { | |
| "epoch": 0.8421052631578947, | |
| "grad_norm": 1.8003302812576294, | |
| "kl": 1.431631326675415, | |
| "learning_rate": 4.1483645377501726e-06, | |
| "logits/chosen": 286558065.7777778, | |
| "logits/rejected": 281730944.0, | |
| "logps/chosen": -683.1265190972222, | |
| "logps/rejected": -394.296875, | |
| "loss": 0.4595, | |
| "rewards/chosen": 0.2565513981713189, | |
| "rewards/margins": 0.4815535848102872, | |
| "rewards/rejected": -0.22500218663896834, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.8631578947368421, | |
| "grad_norm": 1.385386347770691, | |
| "kl": 1.2051702737808228, | |
| "learning_rate": 4.106969024216348e-06, | |
| "logits/chosen": 297838432.0, | |
| "logits/rejected": 317846336.0, | |
| "logps/chosen": -683.0381469726562, | |
| "logps/rejected": -333.70623779296875, | |
| "loss": 0.4735, | |
| "rewards/chosen": 0.30369484424591064, | |
| "rewards/margins": 0.38917434215545654, | |
| "rewards/rejected": -0.0854794979095459, | |
| "step": 41 | |
| }, | |
| { | |
| "epoch": 0.8842105263157894, | |
| "grad_norm": 2.4110379219055176, | |
| "kl": 1.4937351942062378, | |
| "learning_rate": 4.064808680460149e-06, | |
| "logits/chosen": 275613819.5862069, | |
| "logits/rejected": 329032821.0285714, | |
| "logps/chosen": -821.4407327586207, | |
| "logps/rejected": -363.7448660714286, | |
| "loss": 0.4399, | |
| "rewards/chosen": 0.4741830825805664, | |
| "rewards/margins": 0.6639985765729631, | |
| "rewards/rejected": -0.18981549399239675, | |
| "step": 42 | |
| }, | |
| { | |
| "epoch": 0.9052631578947369, | |
| "grad_norm": 1.4786880016326904, | |
| "kl": 1.3580402135849, | |
| "learning_rate": 4.021903572521802e-06, | |
| "logits/chosen": 292805532.9032258, | |
| "logits/rejected": 320448667.1515151, | |
| "logps/chosen": -615.5443548387096, | |
| "logps/rejected": -311.78767163825756, | |
| "loss": 0.4669, | |
| "rewards/chosen": 0.3574201829971806, | |
| "rewards/margins": 0.4636997006622344, | |
| "rewards/rejected": -0.10627951766505386, | |
| "step": 43 | |
| }, | |
| { | |
| "epoch": 0.9263157894736842, | |
| "grad_norm": 1.0773484706878662, | |
| "kl": 1.1364717483520508, | |
| "learning_rate": 3.978274120908957e-06, | |
| "logits/chosen": 298700241.45454544, | |
| "logits/rejected": 323973846.7096774, | |
| "logps/chosen": -521.9517045454545, | |
| "logps/rejected": -351.1326864919355, | |
| "loss": 0.4819, | |
| "rewards/chosen": 0.21399710395119406, | |
| "rewards/margins": 0.30352772575669273, | |
| "rewards/rejected": -0.08953062180549867, | |
| "step": 44 | |
| }, | |
| { | |
| "epoch": 0.9473684210526315, | |
| "grad_norm": 1.8579978942871094, | |
| "kl": 1.5150351524353027, | |
| "learning_rate": 3.933941090877615e-06, | |
| "logits/chosen": 250053461.33333334, | |
| "logits/rejected": 297370359.7419355, | |
| "logps/chosen": -674.4289180871212, | |
| "logps/rejected": -328.01861769153226, | |
| "loss": 0.4562, | |
| "rewards/chosen": 0.3749615929343484, | |
| "rewards/margins": 0.5518351999545726, | |
| "rewards/rejected": -0.17687360702022428, | |
| "step": 45 | |
| }, | |
| { | |
| "epoch": 0.968421052631579, | |
| "grad_norm": 1.442551612854004, | |
| "kl": 1.4570834636688232, | |
| "learning_rate": 3.888925582549006e-06, | |
| "logits/chosen": 296643956.3636364, | |
| "logits/rejected": 300621658.83870965, | |
| "logps/chosen": -726.8507339015151, | |
| "logps/rejected": -310.6574470766129, | |
| "loss": 0.4584, | |
| "rewards/chosen": 0.3162169022993608, | |
| "rewards/margins": 0.4777500119027504, | |
| "rewards/rejected": -0.16153310960338962, | |
| "step": 46 | |
| }, | |
| { | |
| "epoch": 0.9894736842105263, | |
| "grad_norm": 1.5954558849334717, | |
| "kl": 1.3480093479156494, | |
| "learning_rate": 3.8432490208670605e-06, | |
| "logits/chosen": 266226656.96969697, | |
| "logits/rejected": 327847011.0967742, | |
| "logps/chosen": -818.7182765151515, | |
| "logps/rejected": -304.5207283266129, | |
| "loss": 0.4383, | |
| "rewards/chosen": 0.4229244463371508, | |
| "rewards/margins": 0.5965663433541067, | |
| "rewards/rejected": -0.17364189701695595, | |
| "step": 47 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "grad_norm": 0.6833202838897705, | |
| "kl": 0.490053653717041, | |
| "learning_rate": 3.796933145401304e-06, | |
| "logits/chosen": 347558865.45454544, | |
| "logits/rejected": 323800656.84210527, | |
| "logps/chosen": -481.98979048295456, | |
| "logps/rejected": -379.3189761513158, | |
| "loss": 0.231, | |
| "rewards/chosen": 0.20139399441805753, | |
| "rewards/margins": 0.35350871200196476, | |
| "rewards/rejected": -0.15211471758390727, | |
| "step": 48 | |
| }, | |
| { | |
| "epoch": 1.0210526315789474, | |
| "grad_norm": 1.429196834564209, | |
| "kl": 2.1841373443603516, | |
| "learning_rate": 3.7500000000000005e-06, | |
| "logits/chosen": 271206684.4444444, | |
| "logits/rejected": 308082724.5714286, | |
| "logps/chosen": -739.5323893229166, | |
| "logps/rejected": -329.6499720982143, | |
| "loss": 0.452, | |
| "rewards/chosen": 0.4243852562374539, | |
| "rewards/margins": 0.6468354274356176, | |
| "rewards/rejected": -0.2224501711981637, | |
| "step": 49 | |
| }, | |
| { | |
| "epoch": 1.0421052631578946, | |
| "grad_norm": 1.9531340599060059, | |
| "kl": 2.2712414264678955, | |
| "learning_rate": 3.7024719222984696e-06, | |
| "logits/chosen": 283599008.0, | |
| "logits/rejected": 300947008.0, | |
| "logps/chosen": -664.508056640625, | |
| "logps/rejected": -342.87835693359375, | |
| "loss": 0.4543, | |
| "rewards/chosen": 0.5367215871810913, | |
| "rewards/margins": 0.7264900505542755, | |
| "rewards/rejected": -0.1897684633731842, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 1.063157894736842, | |
| "grad_norm": 1.3136494159698486, | |
| "kl": 1.6561638116836548, | |
| "learning_rate": 3.654371533087586e-06, | |
| "logits/chosen": 269414235.4285714, | |
| "logits/rejected": 305372387.5555556, | |
| "logps/chosen": -808.3594447544643, | |
| "logps/rejected": -321.4450412326389, | |
| "loss": 0.4417, | |
| "rewards/chosen": 0.4906304563794817, | |
| "rewards/margins": 0.7233401290954106, | |
| "rewards/rejected": -0.23270967271592882, | |
| "step": 51 | |
| }, | |
| { | |
| "epoch": 1.0842105263157895, | |
| "grad_norm": 2.3089241981506348, | |
| "kl": 3.5582315921783447, | |
| "learning_rate": 3.6057217255475034e-06, | |
| "logits/chosen": 253091982.2222222, | |
| "logits/rejected": 307139657.14285713, | |
| "logps/chosen": -741.5936957465278, | |
| "logps/rejected": -328.35609654017856, | |
| "loss": 0.4255, | |
| "rewards/chosen": 0.7155967818366157, | |
| "rewards/margins": 1.0210523643190899, | |
| "rewards/rejected": -0.3054555824824742, | |
| "step": 52 | |
| }, | |
| { | |
| "epoch": 1.1052631578947367, | |
| "grad_norm": 1.4406310319900513, | |
| "kl": 2.404527425765991, | |
| "learning_rate": 3.556545654351749e-06, | |
| "logits/chosen": 293073600.0, | |
| "logits/rejected": 317258272.0, | |
| "logps/chosen": -809.40478515625, | |
| "logps/rejected": -338.89068603515625, | |
| "loss": 0.4662, | |
| "rewards/chosen": 0.570232629776001, | |
| "rewards/margins": 0.706758126616478, | |
| "rewards/rejected": -0.136525496840477, | |
| "step": 53 | |
| }, | |
| { | |
| "epoch": 1.1263157894736842, | |
| "grad_norm": 0.8961158394813538, | |
| "kl": 0.712938666343689, | |
| "learning_rate": 3.5068667246468437e-06, | |
| "logits/chosen": 307262500.5714286, | |
| "logits/rejected": 325764835.5555556, | |
| "logps/chosen": -476.66078404017856, | |
| "logps/rejected": -288.21210394965277, | |
| "loss": 0.4594, | |
| "rewards/chosen": 0.2979165826525007, | |
| "rewards/margins": 0.4678047744054643, | |
| "rewards/rejected": -0.1698881917529636, | |
| "step": 54 | |
| }, | |
| { | |
| "epoch": 1.1473684210526316, | |
| "grad_norm": 1.1575278043746948, | |
| "kl": 1.8655436038970947, | |
| "learning_rate": 3.4567085809127247e-06, | |
| "logits/chosen": 305821866.6666667, | |
| "logits/rejected": 310424832.0, | |
| "logps/chosen": -604.8497721354166, | |
| "logps/rejected": -279.33816964285717, | |
| "loss": 0.4503, | |
| "rewards/chosen": 0.44473473230997723, | |
| "rewards/margins": 0.6642032237279983, | |
| "rewards/rejected": -0.21946849141802108, | |
| "step": 55 | |
| }, | |
| { | |
| "epoch": 1.168421052631579, | |
| "grad_norm": 1.4242761135101318, | |
| "kl": 2.379883289337158, | |
| "learning_rate": 3.406095095709254e-06, | |
| "logits/chosen": 300870336.0, | |
| "logits/rejected": 338083168.0, | |
| "logps/chosen": -594.4110107421875, | |
| "logps/rejected": -323.47467041015625, | |
| "loss": 0.4548, | |
| "rewards/chosen": 0.4931264817714691, | |
| "rewards/margins": 0.7279564440250397, | |
| "rewards/rejected": -0.23482996225357056, | |
| "step": 56 | |
| }, | |
| { | |
| "epoch": 1.1894736842105262, | |
| "grad_norm": 1.1814433336257935, | |
| "kl": 1.7233543395996094, | |
| "learning_rate": 3.3550503583141726e-06, | |
| "logits/chosen": 281220224.0, | |
| "logits/rejected": 310704924.4444444, | |
| "logps/chosen": -652.3648158482143, | |
| "logps/rejected": -317.59087456597223, | |
| "loss": 0.4316, | |
| "rewards/chosen": 0.4687236377171108, | |
| "rewards/margins": 0.7624165973966084, | |
| "rewards/rejected": -0.2936929596794976, | |
| "step": 57 | |
| }, | |
| { | |
| "epoch": 1.2105263157894737, | |
| "grad_norm": 1.7414805889129639, | |
| "kl": 3.008741855621338, | |
| "learning_rate": 3.303598663257904e-06, | |
| "logits/chosen": 240169557.33333334, | |
| "logits/rejected": 302531145.14285713, | |
| "logps/chosen": -752.9740668402778, | |
| "logps/rejected": -296.3207310267857, | |
| "loss": 0.4323, | |
| "rewards/chosen": 0.6310732099745009, | |
| "rewards/margins": 0.9700051110888285, | |
| "rewards/rejected": -0.3389319011143276, | |
| "step": 58 | |
| }, | |
| { | |
| "epoch": 1.231578947368421, | |
| "grad_norm": 1.3317354917526245, | |
| "kl": 2.0062644481658936, | |
| "learning_rate": 3.2517644987606827e-06, | |
| "logits/chosen": 287379040.0, | |
| "logits/rejected": 301314176.0, | |
| "logps/chosen": -622.8872680664062, | |
| "logps/rejected": -286.7091064453125, | |
| "loss": 0.4326, | |
| "rewards/chosen": 0.49011722207069397, | |
| "rewards/margins": 0.9077150523662567, | |
| "rewards/rejected": -0.41759783029556274, | |
| "step": 59 | |
| }, | |
| { | |
| "epoch": 1.2526315789473683, | |
| "grad_norm": 1.2843542098999023, | |
| "kl": 1.9442932605743408, | |
| "learning_rate": 3.199572535077481e-06, | |
| "logits/chosen": 284359917.71428573, | |
| "logits/rejected": 299180487.1111111, | |
| "logps/chosen": -695.6732700892857, | |
| "logps/rejected": -348.7754177517361, | |
| "loss": 0.4422, | |
| "rewards/chosen": 0.5740087372916085, | |
| "rewards/margins": 0.8209883712586902, | |
| "rewards/rejected": -0.2469796339670817, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 1.2736842105263158, | |
| "grad_norm": 1.2031817436218262, | |
| "kl": 2.3849363327026367, | |
| "learning_rate": 3.147047612756302e-06, | |
| "logits/chosen": 303765760.0, | |
| "logits/rejected": 314450102.85714287, | |
| "logps/chosen": -649.5403103298611, | |
| "logps/rejected": -333.43558175223217, | |
| "loss": 0.4408, | |
| "rewards/chosen": 0.5053782992892795, | |
| "rewards/margins": 0.8684476292322552, | |
| "rewards/rejected": -0.36306932994297575, | |
| "step": 61 | |
| }, | |
| { | |
| "epoch": 1.2947368421052632, | |
| "grad_norm": 1.274133563041687, | |
| "kl": 1.325626254081726, | |
| "learning_rate": 3.094214730815433e-06, | |
| "logits/chosen": 278172608.0, | |
| "logits/rejected": 290380704.0, | |
| "logps/chosen": -515.3540649414062, | |
| "logps/rejected": -267.65130615234375, | |
| "loss": 0.4527, | |
| "rewards/chosen": 0.4884217381477356, | |
| "rewards/margins": 0.6718673706054688, | |
| "rewards/rejected": -0.18344563245773315, | |
| "step": 62 | |
| }, | |
| { | |
| "epoch": 1.3157894736842106, | |
| "grad_norm": 1.1882749795913696, | |
| "kl": 1.1256272792816162, | |
| "learning_rate": 3.0410990348452572e-06, | |
| "logits/chosen": 294844745.14285713, | |
| "logits/rejected": 305883420.4444444, | |
| "logps/chosen": -496.18729073660717, | |
| "logps/rejected": -349.1206325954861, | |
| "loss": 0.4201, | |
| "rewards/chosen": 0.5389730930328369, | |
| "rewards/margins": 1.0107019477420383, | |
| "rewards/rejected": -0.4717288547092014, | |
| "step": 63 | |
| }, | |
| { | |
| "epoch": 1.3368421052631578, | |
| "grad_norm": 1.1736260652542114, | |
| "kl": 2.3068413734436035, | |
| "learning_rate": 2.9877258050403214e-06, | |
| "logits/chosen": 291604650.6666667, | |
| "logits/rejected": 318462098.28571427, | |
| "logps/chosen": -707.8668619791666, | |
| "logps/rejected": -299.36582728794644, | |
| "loss": 0.4267, | |
| "rewards/chosen": 0.638993157280816, | |
| "rewards/margins": 1.0095570049588642, | |
| "rewards/rejected": -0.3705638476780483, | |
| "step": 64 | |
| }, | |
| { | |
| "epoch": 1.3578947368421053, | |
| "grad_norm": 1.2752070426940918, | |
| "kl": 2.008157253265381, | |
| "learning_rate": 2.9341204441673267e-06, | |
| "logits/chosen": 281887584.0, | |
| "logits/rejected": 296646016.0, | |
| "logps/chosen": -544.8499145507812, | |
| "logps/rejected": -338.8497619628906, | |
| "loss": 0.4418, | |
| "rewards/chosen": 0.4943011403083801, | |
| "rewards/margins": 0.862060546875, | |
| "rewards/rejected": -0.3677594065666199, | |
| "step": 65 | |
| }, | |
| { | |
| "epoch": 1.3789473684210527, | |
| "grad_norm": 1.292263150215149, | |
| "kl": 2.7316248416900635, | |
| "learning_rate": 2.880308465474792e-06, | |
| "logits/chosen": 280081645.71428573, | |
| "logits/rejected": 302065777.7777778, | |
| "logps/chosen": -623.3849748883929, | |
| "logps/rejected": -363.41002061631946, | |
| "loss": 0.4141, | |
| "rewards/chosen": 0.6942225183759417, | |
| "rewards/margins": 1.0479256463429285, | |
| "rewards/rejected": -0.3537031279669868, | |
| "step": 66 | |
| }, | |
| { | |
| "epoch": 1.4, | |
| "grad_norm": 1.6679728031158447, | |
| "kl": 3.688748836517334, | |
| "learning_rate": 2.82631548055013e-06, | |
| "logits/chosen": 279772216.8888889, | |
| "logits/rejected": 309287862.85714287, | |
| "logps/chosen": -730.6381293402778, | |
| "logps/rejected": -341.89644949776783, | |
| "loss": 0.4255, | |
| "rewards/chosen": 0.8394391271803114, | |
| "rewards/margins": 1.263045250423371, | |
| "rewards/rejected": -0.4236061232430594, | |
| "step": 67 | |
| }, | |
| { | |
| "epoch": 1.4210526315789473, | |
| "grad_norm": 1.2063454389572144, | |
| "kl": 2.190333366394043, | |
| "learning_rate": 2.7721671871299115e-06, | |
| "logits/chosen": 284919776.0, | |
| "logits/rejected": 314707328.0, | |
| "logps/chosen": -701.7225952148438, | |
| "logps/rejected": -319.76239013671875, | |
| "loss": 0.4379, | |
| "rewards/chosen": 0.6084889769554138, | |
| "rewards/margins": 0.9714170694351196, | |
| "rewards/rejected": -0.3629280924797058, | |
| "step": 68 | |
| }, | |
| { | |
| "epoch": 1.4421052631578948, | |
| "grad_norm": 1.0976450443267822, | |
| "kl": 3.1681458950042725, | |
| "learning_rate": 2.717889356869146e-06, | |
| "logits/chosen": 288710180.5714286, | |
| "logits/rejected": 303286584.8888889, | |
| "logps/chosen": -564.9296177455357, | |
| "logps/rejected": -287.56187608506946, | |
| "loss": 0.41, | |
| "rewards/chosen": 0.8446392331804548, | |
| "rewards/margins": 1.333316681877015, | |
| "rewards/rejected": -0.4886774486965603, | |
| "step": 69 | |
| }, | |
| { | |
| "epoch": 1.4631578947368422, | |
| "grad_norm": 1.4227020740509033, | |
| "kl": 2.573333263397217, | |
| "learning_rate": 2.663507823075358e-06, | |
| "logits/chosen": 292599864.8888889, | |
| "logits/rejected": 310929920.0, | |
| "logps/chosen": -662.43212890625, | |
| "logps/rejected": -329.9785853794643, | |
| "loss": 0.4136, | |
| "rewards/chosen": 0.6803323957655165, | |
| "rewards/margins": 1.2630558770800393, | |
| "rewards/rejected": -0.5827234813145229, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 1.4842105263157894, | |
| "grad_norm": 1.2122981548309326, | |
| "kl": 2.633822202682495, | |
| "learning_rate": 2.6090484684133406e-06, | |
| "logits/chosen": 282540576.0, | |
| "logits/rejected": 311218656.0, | |
| "logps/chosen": -676.41015625, | |
| "logps/rejected": -336.9644775390625, | |
| "loss": 0.4216, | |
| "rewards/chosen": 0.7093576192855835, | |
| "rewards/margins": 1.2067002058029175, | |
| "rewards/rejected": -0.497342586517334, | |
| "step": 71 | |
| }, | |
| { | |
| "epoch": 1.5052631578947369, | |
| "grad_norm": 1.1785283088684082, | |
| "kl": 2.1220672130584717, | |
| "learning_rate": 2.554537212586403e-06, | |
| "logits/chosen": 241519908.57142857, | |
| "logits/rejected": 314742869.3333333, | |
| "logps/chosen": -754.8909737723214, | |
| "logps/rejected": -293.48790147569446, | |
| "loss": 0.3964, | |
| "rewards/chosen": 0.8560841424124581, | |
| "rewards/margins": 1.278062933967227, | |
| "rewards/rejected": -0.42197879155476886, | |
| "step": 72 | |
| }, | |
| { | |
| "epoch": 1.526315789473684, | |
| "grad_norm": 1.2416644096374512, | |
| "kl": 2.510000467300415, | |
| "learning_rate": 2.5e-06, | |
| "logits/chosen": 295481742.2222222, | |
| "logits/rejected": 317371611.4285714, | |
| "logps/chosen": -715.8388671875, | |
| "logps/rejected": -289.58375767299106, | |
| "loss": 0.4226, | |
| "rewards/chosen": 0.680607000986735, | |
| "rewards/margins": 1.1476976417359852, | |
| "rewards/rejected": -0.46709064074925016, | |
| "step": 73 | |
| }, | |
| { | |
| "epoch": 1.5473684210526315, | |
| "grad_norm": 0.9503583908081055, | |
| "kl": 1.7678486108779907, | |
| "learning_rate": 2.4454627874135976e-06, | |
| "logits/chosen": 294551104.0, | |
| "logits/rejected": 318992832.0, | |
| "logps/chosen": -464.9443359375, | |
| "logps/rejected": -306.6573181152344, | |
| "loss": 0.4345, | |
| "rewards/chosen": 0.5198342204093933, | |
| "rewards/margins": 0.9660733342170715, | |
| "rewards/rejected": -0.4462391138076782, | |
| "step": 74 | |
| }, | |
| { | |
| "epoch": 1.568421052631579, | |
| "grad_norm": 1.0121111869812012, | |
| "kl": 3.102855920791626, | |
| "learning_rate": 2.3909515315866606e-06, | |
| "logits/chosen": 253119268.57142857, | |
| "logits/rejected": 307017955.5555556, | |
| "logps/chosen": -744.9547293526786, | |
| "logps/rejected": -303.27305772569446, | |
| "loss": 0.4184, | |
| "rewards/chosen": 0.7782654762268066, | |
| "rewards/margins": 1.2603272332085504, | |
| "rewards/rejected": -0.4820617569817437, | |
| "step": 75 | |
| }, | |
| { | |
| "epoch": 1.5894736842105264, | |
| "grad_norm": 1.0430024862289429, | |
| "kl": 2.240968942642212, | |
| "learning_rate": 2.3364921769246423e-06, | |
| "logits/chosen": 331319495.1111111, | |
| "logits/rejected": 336348635.4285714, | |
| "logps/chosen": -656.6678602430555, | |
| "logps/rejected": -351.60421316964283, | |
| "loss": 0.4397, | |
| "rewards/chosen": 0.5767775111728244, | |
| "rewards/margins": 0.9338111726064531, | |
| "rewards/rejected": -0.35703366143362864, | |
| "step": 76 | |
| }, | |
| { | |
| "epoch": 1.6105263157894738, | |
| "grad_norm": 1.1261143684387207, | |
| "kl": 1.5933310985565186, | |
| "learning_rate": 2.2821106431308546e-06, | |
| "logits/chosen": 278966432.0, | |
| "logits/rejected": 303633792.0, | |
| "logps/chosen": -628.973876953125, | |
| "logps/rejected": -323.4996337890625, | |
| "loss": 0.4327, | |
| "rewards/chosen": 0.4421617388725281, | |
| "rewards/margins": 0.9287151098251343, | |
| "rewards/rejected": -0.4865533709526062, | |
| "step": 77 | |
| }, | |
| { | |
| "epoch": 1.631578947368421, | |
| "grad_norm": 1.2804371118545532, | |
| "kl": 2.7089781761169434, | |
| "learning_rate": 2.2278328128700893e-06, | |
| "logits/chosen": 278770578.28571427, | |
| "logits/rejected": 302573937.7777778, | |
| "logps/chosen": -661.4776785714286, | |
| "logps/rejected": -346.33751085069446, | |
| "loss": 0.4066, | |
| "rewards/chosen": 0.8177859442574638, | |
| "rewards/margins": 1.3722575959705172, | |
| "rewards/rejected": -0.5544716517130533, | |
| "step": 78 | |
| }, | |
| { | |
| "epoch": 1.6526315789473685, | |
| "grad_norm": 0.9231442213058472, | |
| "kl": 2.3125405311584473, | |
| "learning_rate": 2.173684519449872e-06, | |
| "logits/chosen": 310171477.3333333, | |
| "logits/rejected": 305176356.5714286, | |
| "logps/chosen": -482.10584852430554, | |
| "logps/rejected": -341.7206333705357, | |
| "loss": 0.4199, | |
| "rewards/chosen": 0.5320029788547092, | |
| "rewards/margins": 1.0689090168665325, | |
| "rewards/rejected": -0.5369060380118233, | |
| "step": 79 | |
| }, | |
| { | |
| "epoch": 1.6736842105263157, | |
| "grad_norm": 0.8038806319236755, | |
| "kl": 2.2327325344085693, | |
| "learning_rate": 2.1196915345252085e-06, | |
| "logits/chosen": 316725248.0, | |
| "logits/rejected": 324820768.0, | |
| "logps/chosen": -565.4083862304688, | |
| "logps/rejected": -283.5719299316406, | |
| "loss": 0.4239, | |
| "rewards/chosen": 0.606515645980835, | |
| "rewards/margins": 1.2420920133590698, | |
| "rewards/rejected": -0.6355763673782349, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 1.694736842105263, | |
| "grad_norm": 1.22795569896698, | |
| "kl": 3.6404929161071777, | |
| "learning_rate": 2.0658795558326745e-06, | |
| "logits/chosen": 314771090.28571427, | |
| "logits/rejected": 352914432.0, | |
| "logps/chosen": -669.0993303571429, | |
| "logps/rejected": -378.419921875, | |
| "loss": 0.4146, | |
| "rewards/chosen": 0.9939588138035366, | |
| "rewards/margins": 1.4066120480734203, | |
| "rewards/rejected": -0.4126532342698839, | |
| "step": 81 | |
| }, | |
| { | |
| "epoch": 1.7157894736842105, | |
| "grad_norm": 1.2053930759429932, | |
| "kl": 2.856393575668335, | |
| "learning_rate": 2.01227419495968e-06, | |
| "logits/chosen": 281342748.4444444, | |
| "logits/rejected": 315115154.28571427, | |
| "logps/chosen": -597.5861545138889, | |
| "logps/rejected": -316.50892857142856, | |
| "loss": 0.386, | |
| "rewards/chosen": 0.7246790991889106, | |
| "rewards/margins": 1.5443669425116644, | |
| "rewards/rejected": -0.8196878433227539, | |
| "step": 82 | |
| }, | |
| { | |
| "epoch": 1.736842105263158, | |
| "grad_norm": 0.8006544709205627, | |
| "kl": 2.2694008350372314, | |
| "learning_rate": 1.958900965154743e-06, | |
| "logits/chosen": 301532064.0, | |
| "logits/rejected": 307345312.0, | |
| "logps/chosen": -462.2938232421875, | |
| "logps/rejected": -235.0488739013672, | |
| "loss": 0.4387, | |
| "rewards/chosen": 0.4603995978832245, | |
| "rewards/margins": 1.0491024553775787, | |
| "rewards/rejected": -0.5887028574943542, | |
| "step": 83 | |
| }, | |
| { | |
| "epoch": 1.7578947368421054, | |
| "grad_norm": 1.1402519941329956, | |
| "kl": 1.9562792778015137, | |
| "learning_rate": 1.9057852691845677e-06, | |
| "logits/chosen": 272010203.4285714, | |
| "logits/rejected": 308657237.3333333, | |
| "logps/chosen": -570.7827497209821, | |
| "logps/rejected": -313.10308159722223, | |
| "loss": 0.3831, | |
| "rewards/chosen": 0.827030863080706, | |
| "rewards/margins": 1.4214982910761758, | |
| "rewards/rejected": -0.5944674279954698, | |
| "step": 84 | |
| }, | |
| { | |
| "epoch": 1.7789473684210526, | |
| "grad_norm": 1.1353209018707275, | |
| "kl": 3.1249067783355713, | |
| "learning_rate": 1.852952387243698e-06, | |
| "logits/chosen": 282185585.7777778, | |
| "logits/rejected": 323195904.0, | |
| "logps/chosen": -710.0297309027778, | |
| "logps/rejected": -292.21461704799106, | |
| "loss": 0.3872, | |
| "rewards/chosen": 0.8977683385213217, | |
| "rewards/margins": 1.7488488242739724, | |
| "rewards/rejected": -0.8510804857526507, | |
| "step": 85 | |
| }, | |
| { | |
| "epoch": 1.8, | |
| "grad_norm": 1.0260812044143677, | |
| "kl": 1.7017306089401245, | |
| "learning_rate": 1.8004274649225201e-06, | |
| "logits/chosen": 291513664.0, | |
| "logits/rejected": 314734592.0, | |
| "logps/chosen": -544.3753662109375, | |
| "logps/rejected": -303.4842529296875, | |
| "loss": 0.4086, | |
| "rewards/chosen": 0.6065320372581482, | |
| "rewards/margins": 1.2060632109642029, | |
| "rewards/rejected": -0.5995311737060547, | |
| "step": 86 | |
| }, | |
| { | |
| "epoch": 1.8210526315789473, | |
| "grad_norm": 1.1943074464797974, | |
| "kl": 3.096353769302368, | |
| "learning_rate": 1.7482355012393177e-06, | |
| "logits/chosen": 308626578.28571427, | |
| "logits/rejected": 320611356.4444444, | |
| "logps/chosen": -635.6044921875, | |
| "logps/rejected": -328.5426974826389, | |
| "loss": 0.3767, | |
| "rewards/chosen": 0.8776694025312152, | |
| "rewards/margins": 1.486631302606492, | |
| "rewards/rejected": -0.6089619000752767, | |
| "step": 87 | |
| }, | |
| { | |
| "epoch": 1.8421052631578947, | |
| "grad_norm": 1.3380628824234009, | |
| "kl": 2.669947385787964, | |
| "learning_rate": 1.6964013367420967e-06, | |
| "logits/chosen": 289058076.4444444, | |
| "logits/rejected": 283605193.14285713, | |
| "logps/chosen": -677.7171223958334, | |
| "logps/rejected": -403.6229771205357, | |
| "loss": 0.3675, | |
| "rewards/chosen": 0.7974962128533257, | |
| "rewards/margins": 1.9551072423420255, | |
| "rewards/rejected": -1.1576110294886999, | |
| "step": 88 | |
| }, | |
| { | |
| "epoch": 1.8631578947368421, | |
| "grad_norm": 1.051780343055725, | |
| "kl": 2.4696009159088135, | |
| "learning_rate": 1.6449496416858285e-06, | |
| "logits/chosen": 300929024.0, | |
| "logits/rejected": 320133600.0, | |
| "logps/chosen": -677.1189575195312, | |
| "logps/rejected": -339.54608154296875, | |
| "loss": 0.4024, | |
| "rewards/chosen": 0.8956129550933838, | |
| "rewards/margins": 1.5650758743286133, | |
| "rewards/rejected": -0.6694629192352295, | |
| "step": 89 | |
| }, | |
| { | |
| "epoch": 1.8842105263157896, | |
| "grad_norm": 1.425748348236084, | |
| "kl": 2.594327926635742, | |
| "learning_rate": 1.5939049042907463e-06, | |
| "logits/chosen": 279002712.2758621, | |
| "logits/rejected": 331318096.45714283, | |
| "logps/chosen": -812.5680226293103, | |
| "logps/rejected": -373.0890345982143, | |
| "loss": 0.3106, | |
| "rewards/chosen": 1.3614521684317753, | |
| "rewards/margins": 2.4856834524370766, | |
| "rewards/rejected": -1.1242312840053013, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 1.905263157894737, | |
| "grad_norm": 0.9643742442131042, | |
| "kl": 2.9081778526306152, | |
| "learning_rate": 1.5432914190872757e-06, | |
| "logits/chosen": 295409465.8064516, | |
| "logits/rejected": 322480624.4848485, | |
| "logps/chosen": -609.1846018145161, | |
| "logps/rejected": -318.19146543560606, | |
| "loss": 0.381, | |
| "rewards/chosen": 0.993392082952684, | |
| "rewards/margins": 1.7400533558686109, | |
| "rewards/rejected": -0.7466612729159269, | |
| "step": 91 | |
| }, | |
| { | |
| "epoch": 1.9263157894736842, | |
| "grad_norm": 0.9330692291259766, | |
| "kl": 2.2181286811828613, | |
| "learning_rate": 1.4931332753531575e-06, | |
| "logits/chosen": 301478136.24242425, | |
| "logits/rejected": 325849946.83870965, | |
| "logps/chosen": -518.4118726325758, | |
| "logps/rejected": -356.10846144153226, | |
| "loss": 0.4365, | |
| "rewards/chosen": 0.5679830493349017, | |
| "rewards/margins": 1.1550928304155783, | |
| "rewards/rejected": -0.5871097810806767, | |
| "step": 92 | |
| }, | |
| { | |
| "epoch": 1.9473684210526314, | |
| "grad_norm": 1.2292976379394531, | |
| "kl": 3.2448878288269043, | |
| "learning_rate": 1.443454345648252e-06, | |
| "logits/chosen": 252509028.84848484, | |
| "logits/rejected": 298726697.2903226, | |
| "logps/chosen": -668.6770833333334, | |
| "logps/rejected": -336.6697013608871, | |
| "loss": 0.3723, | |
| "rewards/chosen": 0.950144623265122, | |
| "rewards/margins": 1.9921284430420876, | |
| "rewards/rejected": -1.0419838197769657, | |
| "step": 93 | |
| }, | |
| { | |
| "epoch": 1.9684210526315788, | |
| "grad_norm": 1.0690096616744995, | |
| "kl": 2.7862634658813477, | |
| "learning_rate": 1.3942782744524974e-06, | |
| "logits/chosen": 298993601.93939394, | |
| "logits/rejected": 302201723.87096775, | |
| "logps/chosen": -721.709990530303, | |
| "logps/rejected": -317.60660282258067, | |
| "loss": 0.3709, | |
| "rewards/chosen": 0.8302892049153646, | |
| "rewards/margins": 1.68674196222777, | |
| "rewards/rejected": -0.8564527573124054, | |
| "step": 94 | |
| }, | |
| { | |
| "epoch": 1.9894736842105263, | |
| "grad_norm": 0.9422377943992615, | |
| "kl": 2.6181588172912598, | |
| "learning_rate": 1.3456284669124159e-06, | |
| "logits/chosen": 269021277.09090906, | |
| "logits/rejected": 329800010.32258064, | |
| "logps/chosen": -812.2279829545455, | |
| "logps/rejected": -312.94632056451616, | |
| "loss": 0.3429, | |
| "rewards/chosen": 1.0719468087861033, | |
| "rewards/margins": 2.088143951848572, | |
| "rewards/rejected": -1.0161971430624686, | |
| "step": 95 | |
| }, | |
| { | |
| "epoch": 2.0, | |
| "grad_norm": 0.6018558144569397, | |
| "kl": 0.3343966007232666, | |
| "learning_rate": 1.2975280777015315e-06, | |
| "logits/chosen": 350389713.45454544, | |
| "logits/rejected": 325354549.8947368, | |
| "logps/chosen": -479.17649147727275, | |
| "logps/rejected": -385.98116262335526, | |
| "loss": 0.186, | |
| "rewards/chosen": 0.4827239296653054, | |
| "rewards/margins": 1.3010570535249117, | |
| "rewards/rejected": -0.8183331238596063, | |
| "step": 96 | |
| }, | |
| { | |
| "epoch": 2.0210526315789474, | |
| "grad_norm": 0.9914832711219788, | |
| "kl": 4.49602746963501, | |
| "learning_rate": 1.2500000000000007e-06, | |
| "logits/chosen": 273368320.0, | |
| "logits/rejected": 309601865.14285713, | |
| "logps/chosen": -734.2335069444445, | |
| "logps/rejected": -337.99769810267856, | |
| "loss": 0.3757, | |
| "rewards/chosen": 0.9542705747816298, | |
| "rewards/margins": 2.011491783081539, | |
| "rewards/rejected": -1.0572212082999093, | |
| "step": 97 | |
| }, | |
| { | |
| "epoch": 2.042105263157895, | |
| "grad_norm": 1.185113787651062, | |
| "kl": 4.397549629211426, | |
| "learning_rate": 1.203066854598696e-06, | |
| "logits/chosen": 285693728.0, | |
| "logits/rejected": 302816000.0, | |
| "logps/chosen": -658.1407470703125, | |
| "logps/rejected": -351.9900207519531, | |
| "loss": 0.3642, | |
| "rewards/chosen": 1.173454761505127, | |
| "rewards/margins": 2.274389863014221, | |
| "rewards/rejected": -1.1009351015090942, | |
| "step": 98 | |
| }, | |
| { | |
| "epoch": 2.0631578947368423, | |
| "grad_norm": 0.975797176361084, | |
| "kl": 2.5340423583984375, | |
| "learning_rate": 1.1567509791329402e-06, | |
| "logits/chosen": 272200941.71428573, | |
| "logits/rejected": 307242979.5555556, | |
| "logps/chosen": -803.5295061383929, | |
| "logps/rejected": -330.32402886284723, | |
| "loss": 0.3608, | |
| "rewards/chosen": 0.9736276354108538, | |
| "rewards/margins": 2.0942357532561773, | |
| "rewards/rejected": -1.1206081178453233, | |
| "step": 99 | |
| }, | |
| { | |
| "epoch": 2.0842105263157893, | |
| "grad_norm": 1.179176688194275, | |
| "kl": 6.256747722625732, | |
| "learning_rate": 1.1110744174509952e-06, | |
| "logits/chosen": 255072796.44444445, | |
| "logits/rejected": 308663661.71428573, | |
| "logps/chosen": -734.2338324652778, | |
| "logps/rejected": -339.58851841517856, | |
| "loss": 0.3406, | |
| "rewards/chosen": 1.4515838623046875, | |
| "rewards/margins": 2.880281448364258, | |
| "rewards/rejected": -1.4286975860595703, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 2.1052631578947367, | |
| "grad_norm": 1.2810099124908447, | |
| "kl": 3.9927051067352295, | |
| "learning_rate": 1.0660589091223854e-06, | |
| "logits/chosen": 296453824.0, | |
| "logits/rejected": 319576800.0, | |
| "logps/chosen": -804.0701904296875, | |
| "logps/rejected": -344.6459655761719, | |
| "loss": 0.4024, | |
| "rewards/chosen": 1.1036909818649292, | |
| "rewards/margins": 1.8157426118850708, | |
| "rewards/rejected": -0.7120516300201416, | |
| "step": 101 | |
| }, | |
| { | |
| "epoch": 2.126315789473684, | |
| "grad_norm": 0.7138159871101379, | |
| "kl": 1.1326825618743896, | |
| "learning_rate": 1.0217258790910447e-06, | |
| "logits/chosen": 309144685.71428573, | |
| "logits/rejected": 327665123.5555556, | |
| "logps/chosen": -474.47987583705356, | |
| "logps/rejected": -293.0481228298611, | |
| "loss": 0.4138, | |
| "rewards/chosen": 0.5160130092075893, | |
| "rewards/margins": 1.1695069434150818, | |
| "rewards/rejected": -0.6534939342074924, | |
| "step": 102 | |
| }, | |
| { | |
| "epoch": 2.1473684210526316, | |
| "grad_norm": 0.8066704869270325, | |
| "kl": 2.314944267272949, | |
| "learning_rate": 9.780964274781984e-07, | |
| "logits/chosen": 307828736.0, | |
| "logits/rejected": 312215113.14285713, | |
| "logps/chosen": -601.6102973090278, | |
| "logps/rejected": -286.4847935267857, | |
| "loss": 0.3993, | |
| "rewards/chosen": 0.7686821089850532, | |
| "rewards/margins": 1.7028116877116855, | |
| "rewards/rejected": -0.9341295787266323, | |
| "step": 103 | |
| }, | |
| { | |
| "epoch": 2.168421052631579, | |
| "grad_norm": 0.9273717403411865, | |
| "kl": 3.7996773719787598, | |
| "learning_rate": 9.351913195398523e-07, | |
| "logits/chosen": 303605472.0, | |
| "logits/rejected": 339657600.0, | |
| "logps/chosen": -590.1983642578125, | |
| "logps/rejected": -330.32666015625, | |
| "loss": 0.3999, | |
| "rewards/chosen": 0.9143981337547302, | |
| "rewards/margins": 1.8344270586967468, | |
| "rewards/rejected": -0.9200289249420166, | |
| "step": 104 | |
| }, | |
| { | |
| "epoch": 2.1894736842105265, | |
| "grad_norm": 0.8671090006828308, | |
| "kl": 2.5950183868408203, | |
| "learning_rate": 8.930309757836517e-07, | |
| "logits/chosen": 283258258.28571427, | |
| "logits/rejected": 312255089.7777778, | |
| "logps/chosen": -648.32666015625, | |
| "logps/rejected": -325.6108127170139, | |
| "loss": 0.3448, | |
| "rewards/chosen": 0.8725461278642926, | |
| "rewards/margins": 1.9682320867265974, | |
| "rewards/rejected": -1.0956859588623047, | |
| "step": 105 | |
| }, | |
| { | |
| "epoch": 2.2105263157894735, | |
| "grad_norm": 1.1705127954483032, | |
| "kl": 4.775852203369141, | |
| "learning_rate": 8.516354622498279e-07, | |
| "logits/chosen": 242087224.8888889, | |
| "logits/rejected": 304067876.5714286, | |
| "logps/chosen": -748.5776909722222, | |
| "logps/rejected": -303.85107421875, | |
| "loss": 0.3737, | |
| "rewards/chosen": 1.0707118776109483, | |
| "rewards/margins": 2.1626787109980508, | |
| "rewards/rejected": -1.0919668333871024, | |
| "step": 106 | |
| }, | |
| { | |
| "epoch": 2.231578947368421, | |
| "grad_norm": 0.8211517930030823, | |
| "kl": 2.75072979927063, | |
| "learning_rate": 8.110244809608494e-07, | |
| "logits/chosen": 289833088.0, | |
| "logits/rejected": 302552512.0, | |
| "logps/chosen": -619.5748291015625, | |
| "logps/rejected": -296.18695068359375, | |
| "loss": 0.3668, | |
| "rewards/chosen": 0.8213632702827454, | |
| "rewards/margins": 2.1867424845695496, | |
| "rewards/rejected": -1.3653792142868042, | |
| "step": 107 | |
| }, | |
| { | |
| "epoch": 2.2526315789473683, | |
| "grad_norm": 1.093108892440796, | |
| "kl": 3.059412956237793, | |
| "learning_rate": 7.712173605443269e-07, | |
| "logits/chosen": 285983762.28571427, | |
| "logits/rejected": 300334563.5555556, | |
| "logps/chosen": -692.2897600446429, | |
| "logps/rejected": -355.3886447482639, | |
| "loss": 0.376, | |
| "rewards/chosen": 0.9123561041695731, | |
| "rewards/margins": 1.820660742502364, | |
| "rewards/rejected": -0.9083046383327908, | |
| "step": 108 | |
| }, | |
| { | |
| "epoch": 2.2736842105263158, | |
| "grad_norm": 0.8700193762779236, | |
| "kl": 3.397562026977539, | |
| "learning_rate": 7.322330470336314e-07, | |
| "logits/chosen": 306071779.5555556, | |
| "logits/rejected": 316075373.71428573, | |
| "logps/chosen": -645.9636501736111, | |
| "logps/rejected": -340.6162806919643, | |
| "loss": 0.3865, | |
| "rewards/chosen": 0.863048235575358, | |
| "rewards/margins": 1.944187255132766, | |
| "rewards/rejected": -1.081139019557408, | |
| "step": 109 | |
| }, | |
| { | |
| "epoch": 2.294736842105263, | |
| "grad_norm": 0.9735374450683594, | |
| "kl": 1.8307335376739502, | |
| "learning_rate": 6.940900948506113e-07, | |
| "logits/chosen": 280107104.0, | |
| "logits/rejected": 291821312.0, | |
| "logps/chosen": -512.7600708007812, | |
| "logps/rejected": -272.31732177734375, | |
| "loss": 0.4089, | |
| "rewards/chosen": 0.7478194236755371, | |
| "rewards/margins": 1.3978694677352905, | |
| "rewards/rejected": -0.6500500440597534, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 2.3157894736842106, | |
| "grad_norm": 0.8717783689498901, | |
| "kl": 1.6515731811523438, | |
| "learning_rate": 6.568066579746901e-07, | |
| "logits/chosen": 297068635.4285714, | |
| "logits/rejected": 307091399.1111111, | |
| "logps/chosen": -493.14170619419644, | |
| "logps/rejected": -356.8146701388889, | |
| "loss": 0.3656, | |
| "rewards/chosen": 0.8435340608869281, | |
| "rewards/margins": 2.0846710356455, | |
| "rewards/rejected": -1.241136974758572, | |
| "step": 111 | |
| }, | |
| { | |
| "epoch": 2.336842105263158, | |
| "grad_norm": 0.8384011387825012, | |
| "kl": 2.806180715560913, | |
| "learning_rate": 6.204004813025569e-07, | |
| "logits/chosen": 293585009.7777778, | |
| "logits/rejected": 319276032.0, | |
| "logps/chosen": -704.3138020833334, | |
| "logps/rejected": -306.95455496651783, | |
| "loss": 0.374, | |
| "rewards/chosen": 0.99429808722602, | |
| "rewards/margins": 2.1237333539932495, | |
| "rewards/rejected": -1.1294352667672294, | |
| "step": 112 | |
| }, | |
| { | |
| "epoch": 2.3578947368421055, | |
| "grad_norm": 1.0061078071594238, | |
| "kl": 2.242816686630249, | |
| "learning_rate": 5.848888922025553e-07, | |
| "logits/chosen": 284455776.0, | |
| "logits/rejected": 298613120.0, | |
| "logps/chosen": -542.8225708007812, | |
| "logps/rejected": -345.3612060546875, | |
| "loss": 0.3959, | |
| "rewards/chosen": 0.6970374584197998, | |
| "rewards/margins": 1.7159442901611328, | |
| "rewards/rejected": -1.018906831741333, | |
| "step": 113 | |
| }, | |
| { | |
| "epoch": 2.3789473684210525, | |
| "grad_norm": 1.0236337184906006, | |
| "kl": 3.477278232574463, | |
| "learning_rate": 5.50288792267796e-07, | |
| "logits/chosen": 281507035.4285714, | |
| "logits/rejected": 303031580.4444444, | |
| "logps/chosen": -620.07666015625, | |
| "logps/rejected": -369.61480034722223, | |
| "loss": 0.3544, | |
| "rewards/chosen": 1.025052615574428, | |
| "rewards/margins": 1.9992341086978005, | |
| "rewards/rejected": -0.9741814931233724, | |
| "step": 114 | |
| }, | |
| { | |
| "epoch": 2.4, | |
| "grad_norm": 1.2651081085205078, | |
| "kl": 4.5784382820129395, | |
| "learning_rate": 5.166166492719124e-07, | |
| "logits/chosen": 281606286.2222222, | |
| "logits/rejected": 310555867.4285714, | |
| "logps/chosen": -726.4253472222222, | |
| "logps/rejected": -348.73287527901783, | |
| "loss": 0.3689, | |
| "rewards/chosen": 1.2607183456420898, | |
| "rewards/margins": 2.3679630415780206, | |
| "rewards/rejected": -1.1072446959359306, | |
| "step": 115 | |
| }, | |
| { | |
| "epoch": 2.4210526315789473, | |
| "grad_norm": 0.9246846437454224, | |
| "kl": 2.923830032348633, | |
| "learning_rate": 4.838884893312934e-07, | |
| "logits/chosen": 286494016.0, | |
| "logits/rejected": 315544192.0, | |
| "logps/chosen": -698.613525390625, | |
| "logps/rejected": -325.4237365722656, | |
| "loss": 0.3879, | |
| "rewards/chosen": 0.9193922877311707, | |
| "rewards/margins": 1.8484562039375305, | |
| "rewards/rejected": -0.9290639162063599, | |
| "step": 116 | |
| }, | |
| { | |
| "epoch": 2.442105263157895, | |
| "grad_norm": 0.7015478014945984, | |
| "kl": 4.306000232696533, | |
| "learning_rate": 4.5211988927752026e-07, | |
| "logits/chosen": 291081782.85714287, | |
| "logits/rejected": 304304839.1111111, | |
| "logps/chosen": -561.2941545758929, | |
| "logps/rejected": -294.81694878472223, | |
| "loss": 0.3653, | |
| "rewards/chosen": 1.208183969770159, | |
| "rewards/margins": 2.422367989070832, | |
| "rewards/rejected": -1.2141840193006728, | |
| "step": 117 | |
| }, | |
| { | |
| "epoch": 2.463157894736842, | |
| "grad_norm": 1.10671865940094, | |
| "kl": 3.112697124481201, | |
| "learning_rate": 4.2132596924363666e-07, | |
| "logits/chosen": 294296291.5555556, | |
| "logits/rejected": 311906011.4285714, | |
| "logps/chosen": -659.6044921875, | |
| "logps/rejected": -337.13692801339283, | |
| "loss": 0.3648, | |
| "rewards/chosen": 0.9630860222710503, | |
| "rewards/margins": 2.2616418051341225, | |
| "rewards/rejected": -1.298555782863072, | |
| "step": 118 | |
| }, | |
| { | |
| "epoch": 2.4842105263157896, | |
| "grad_norm": 0.9874948859214783, | |
| "kl": 3.2510499954223633, | |
| "learning_rate": 3.915213854677863e-07, | |
| "logits/chosen": 284307872.0, | |
| "logits/rejected": 312279616.0, | |
| "logps/chosen": -673.400634765625, | |
| "logps/rejected": -343.1709289550781, | |
| "loss": 0.3752, | |
| "rewards/chosen": 1.0103015899658203, | |
| "rewards/margins": 2.128287434577942, | |
| "rewards/rejected": -1.1179858446121216, | |
| "step": 119 | |
| }, | |
| { | |
| "epoch": 2.5052631578947366, | |
| "grad_norm": 0.9072942733764648, | |
| "kl": 2.408541202545166, | |
| "learning_rate": 3.627203233176341e-07, | |
| "logits/chosen": 242695186.2857143, | |
| "logits/rejected": 315699996.4444444, | |
| "logps/chosen": -751.4161551339286, | |
| "logps/rejected": -298.6019694010417, | |
| "loss": 0.3405, | |
| "rewards/chosen": 1.203570774623326, | |
| "rewards/margins": 2.136957047477601, | |
| "rewards/rejected": -0.9333862728542752, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 2.526315789473684, | |
| "grad_norm": 1.0498640537261963, | |
| "kl": 2.8332810401916504, | |
| "learning_rate": 3.3493649053890325e-07, | |
| "logits/chosen": 296178346.6666667, | |
| "logits/rejected": 317656722.28571427, | |
| "logps/chosen": -713.7129991319445, | |
| "logps/rejected": -295.0296107700893, | |
| "loss": 0.3833, | |
| "rewards/chosen": 0.8931880527072482, | |
| "rewards/margins": 1.9048690644521562, | |
| "rewards/rejected": -1.011681011744908, | |
| "step": 121 | |
| }, | |
| { | |
| "epoch": 2.5473684210526315, | |
| "grad_norm": 0.7236945033073425, | |
| "kl": 2.3001227378845215, | |
| "learning_rate": 3.081831107312308e-07, | |
| "logits/chosen": 295994208.0, | |
| "logits/rejected": 320040224.0, | |
| "logps/chosen": -463.0760192871094, | |
| "logps/rejected": -311.6647644042969, | |
| "loss": 0.4038, | |
| "rewards/chosen": 0.7066670656204224, | |
| "rewards/margins": 1.6536514163017273, | |
| "rewards/rejected": -0.9469843506813049, | |
| "step": 122 | |
| }, | |
| { | |
| "epoch": 2.568421052631579, | |
| "grad_norm": 0.809329092502594, | |
| "kl": 3.3746235370635986, | |
| "learning_rate": 2.8247291705444575e-07, | |
| "logits/chosen": 254174134.85714287, | |
| "logits/rejected": 307626439.1111111, | |
| "logps/chosen": -742.4175502232143, | |
| "logps/rejected": -308.20654296875, | |
| "loss": 0.3842, | |
| "rewards/chosen": 1.0319812638419015, | |
| "rewards/margins": 2.007392784905812, | |
| "rewards/rejected": -0.9754115210639106, | |
| "step": 123 | |
| }, | |
| { | |
| "epoch": 2.5894736842105264, | |
| "grad_norm": 0.935296356678009, | |
| "kl": 2.1826891899108887, | |
| "learning_rate": 2.5781814616827936e-07, | |
| "logits/chosen": 332361813.3333333, | |
| "logits/rejected": 337299017.14285713, | |
| "logps/chosen": -655.0251736111111, | |
| "logps/rejected": -355.64554268973217, | |
| "loss": 0.4048, | |
| "rewards/chosen": 0.7410464816623263, | |
| "rewards/margins": 1.5022159152560763, | |
| "rewards/rejected": -0.76116943359375, | |
| "step": 124 | |
| }, | |
| { | |
| "epoch": 2.610526315789474, | |
| "grad_norm": 0.984870195388794, | |
| "kl": 1.8854070901870728, | |
| "learning_rate": 2.3423053240837518e-07, | |
| "logits/chosen": 279978112.0, | |
| "logits/rejected": 304146432.0, | |
| "logps/chosen": -627.2296142578125, | |
| "logps/rejected": -328.7203369140625, | |
| "loss": 0.3887, | |
| "rewards/chosen": 0.6165910959243774, | |
| "rewards/margins": 1.625217080116272, | |
| "rewards/rejected": -1.0086259841918945, | |
| "step": 125 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 144, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 25, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": false | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 8, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |