{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9981298423724285,
  "eval_steps": 400,
  "global_step": 467,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0021373230029388193,
      "grad_norm": 8.463839594398154,
      "learning_rate": 1.0638297872340425e-08,
      "logits/chosen": -5.03189754486084,
      "logits/rejected": -4.649786472320557,
      "logps/chosen": -1.325859785079956,
      "logps/rejected": -1.2691845893859863,
      "loss": 1.6361,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -3.3146491050720215,
      "rewards/margins": -0.1416877955198288,
      "rewards/rejected": -3.172961473464966,
      "step": 1
    },
    {
      "epoch": 0.010686615014694095,
      "grad_norm": 10.461652514268042,
      "learning_rate": 5.3191489361702123e-08,
      "logits/chosen": -4.613029479980469,
      "logits/rejected": -4.432791709899902,
      "logps/chosen": -1.3983737230300903,
      "logps/rejected": -1.4233554601669312,
      "loss": 1.8223,
      "rewards/accuracies": 0.453125,
      "rewards/chosen": -3.495934247970581,
      "rewards/margins": 0.0624544657766819,
      "rewards/rejected": -3.5583884716033936,
      "step": 5
    },
    {
      "epoch": 0.02137323002938819,
      "grad_norm": 14.978344124531132,
      "learning_rate": 1.0638297872340425e-07,
      "logits/chosen": -4.634655952453613,
      "logits/rejected": -4.519498348236084,
      "logps/chosen": -1.5365169048309326,
      "logps/rejected": -1.5281957387924194,
      "loss": 1.8105,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -3.8412926197052,
      "rewards/margins": -0.020802771672606468,
      "rewards/rejected": -3.8204894065856934,
      "step": 10
    },
    {
      "epoch": 0.03205984504408229,
      "grad_norm": 9.265143341850845,
      "learning_rate": 1.5957446808510638e-07,
      "logits/chosen": -4.536217212677002,
      "logits/rejected": -4.47308874130249,
      "logps/chosen": -1.5140950679779053,
      "logps/rejected": -1.4939788579940796,
      "loss": 1.748,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -3.785238265991211,
      "rewards/margins": -0.05029074102640152,
      "rewards/rejected": -3.7349472045898438,
      "step": 15
    },
    {
      "epoch": 0.04274646005877638,
      "grad_norm": 7.20742555554014,
      "learning_rate": 2.127659574468085e-07,
      "logits/chosen": -4.679377555847168,
      "logits/rejected": -4.679813385009766,
      "logps/chosen": -1.4990894794464111,
      "logps/rejected": -1.5615761280059814,
      "loss": 1.7569,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -3.7477238178253174,
      "rewards/margins": 0.15621671080589294,
      "rewards/rejected": -3.9039406776428223,
      "step": 20
    },
    {
      "epoch": 0.053433075073470476,
      "grad_norm": 8.314479583512021,
      "learning_rate": 2.659574468085106e-07,
      "logits/chosen": -4.735424041748047,
      "logits/rejected": -4.67797327041626,
      "logps/chosen": -1.5032179355621338,
      "logps/rejected": -1.5036437511444092,
      "loss": 1.7663,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -3.758044719696045,
      "rewards/margins": 0.001064938260242343,
      "rewards/rejected": -3.7591099739074707,
      "step": 25
    },
    {
      "epoch": 0.06411969008816458,
      "grad_norm": 8.24895452301186,
      "learning_rate": 3.1914893617021275e-07,
      "logits/chosen": -4.979925632476807,
      "logits/rejected": -4.7350873947143555,
      "logps/chosen": -1.526336431503296,
      "logps/rejected": -1.5037505626678467,
      "loss": 1.7647,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -3.81584095954895,
      "rewards/margins": -0.05646459385752678,
      "rewards/rejected": -3.7593765258789062,
      "step": 30
    },
    {
      "epoch": 0.07480630510285867,
      "grad_norm": 8.218009051807567,
      "learning_rate": 3.7234042553191484e-07,
      "logits/chosen": -4.755112171173096,
      "logits/rejected": -4.576828479766846,
      "logps/chosen": -1.5506718158721924,
      "logps/rejected": -1.5067684650421143,
      "loss": 1.7863,
      "rewards/accuracies": 0.32499998807907104,
      "rewards/chosen": -3.876680374145508,
      "rewards/margins": -0.10975880920886993,
      "rewards/rejected": -3.7669215202331543,
      "step": 35
    },
    {
      "epoch": 0.08549292011755276,
      "grad_norm": 10.346763541049816,
      "learning_rate": 4.25531914893617e-07,
      "logits/chosen": -4.490562915802002,
      "logits/rejected": -4.430506706237793,
      "logps/chosen": -1.4821277856826782,
      "logps/rejected": -1.5026350021362305,
      "loss": 1.7765,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -3.7053191661834717,
      "rewards/margins": 0.05126795917749405,
      "rewards/rejected": -3.756587266921997,
      "step": 40
    },
    {
      "epoch": 0.09617953513224686,
      "grad_norm": 8.159954026995056,
      "learning_rate": 4.787234042553192e-07,
      "logits/chosen": -4.584813594818115,
      "logits/rejected": -4.397616386413574,
      "logps/chosen": -1.5396957397460938,
      "logps/rejected": -1.5210858583450317,
      "loss": 1.7353,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -3.8492393493652344,
      "rewards/margins": -0.046524763107299805,
      "rewards/rejected": -3.8027145862579346,
      "step": 45
    },
    {
      "epoch": 0.10686615014694095,
      "grad_norm": 6.871700800549116,
      "learning_rate": 4.999370587356267e-07,
      "logits/chosen": -4.505709648132324,
      "logits/rejected": -4.605370998382568,
      "logps/chosen": -1.4193190336227417,
      "logps/rejected": -1.5022988319396973,
      "loss": 1.7108,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -3.54829740524292,
      "rewards/margins": 0.20744964480400085,
      "rewards/rejected": -3.7557475566864014,
      "step": 50
    },
    {
      "epoch": 0.11755276516163506,
      "grad_norm": 8.013382748882483,
      "learning_rate": 4.995525324419337e-07,
      "logits/chosen": -4.650801658630371,
      "logits/rejected": -4.462812900543213,
      "logps/chosen": -1.3039531707763672,
      "logps/rejected": -1.3392069339752197,
      "loss": 1.7299,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -3.259882688522339,
      "rewards/margins": 0.08813469856977463,
      "rewards/rejected": -3.348017454147339,
      "step": 55
    },
    {
      "epoch": 0.12823938017632916,
      "grad_norm": 8.303134763756917,
      "learning_rate": 4.988189843662815e-07,
      "logits/chosen": -4.751210689544678,
      "logits/rejected": -4.686429023742676,
      "logps/chosen": -1.2432037591934204,
      "logps/rejected": -1.2768182754516602,
      "loss": 1.7359,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -3.1080093383789062,
      "rewards/margins": 0.08403660356998444,
      "rewards/rejected": -3.1920459270477295,
      "step": 60
    },
    {
      "epoch": 0.13892599519102325,
      "grad_norm": 6.777024673739444,
      "learning_rate": 4.977374404419837e-07,
      "logits/chosen": -4.5330071449279785,
      "logits/rejected": -4.71773099899292,
      "logps/chosen": -1.3066389560699463,
      "logps/rejected": -1.2977256774902344,
      "loss": 1.7457,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -3.266597270965576,
      "rewards/margins": -0.02228318527340889,
      "rewards/rejected": -3.244314193725586,
      "step": 65
    },
    {
      "epoch": 0.14961261020571734,
      "grad_norm": 9.380383365435003,
      "learning_rate": 4.963094133060148e-07,
      "logits/chosen": -4.556839466094971,
      "logits/rejected": -4.482637882232666,
      "logps/chosen": -1.3503261804580688,
      "logps/rejected": -1.309463620185852,
      "loss": 1.7783,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": -3.3758158683776855,
      "rewards/margins": -0.1021568551659584,
      "rewards/rejected": -3.2736592292785645,
      "step": 70
    },
    {
      "epoch": 0.16029922522041143,
      "grad_norm": 8.316182420752773,
      "learning_rate": 4.945369001834514e-07,
      "logits/chosen": -4.512010097503662,
      "logits/rejected": -4.473996639251709,
      "logps/chosen": -1.3320636749267578,
      "logps/rejected": -1.3820534944534302,
      "loss": 1.6619,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -3.3301587104797363,
      "rewards/margins": 0.12497501075267792,
      "rewards/rejected": -3.455134153366089,
      "step": 75
    },
    {
      "epoch": 0.17098584023510552,
      "grad_norm": 7.968870017741486,
      "learning_rate": 4.924223800941717e-07,
      "logits/chosen": -4.695282936096191,
      "logits/rejected": -4.676292419433594,
      "logps/chosen": -1.3696571588516235,
      "logps/rejected": -1.341069221496582,
      "loss": 1.7283,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -3.4241433143615723,
      "rewards/margins": -0.0714699923992157,
      "rewards/rejected": -3.352673053741455,
      "step": 80
    },
    {
      "epoch": 0.18167245524979964,
      "grad_norm": 6.985848218948992,
      "learning_rate": 4.899688103857222e-07,
      "logits/chosen": -4.516369342803955,
      "logits/rejected": -4.401045799255371,
      "logps/chosen": -1.3444452285766602,
      "logps/rejected": -1.356410264968872,
      "loss": 1.7358,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -3.3611133098602295,
      "rewards/margins": 0.02991255186498165,
      "rewards/rejected": -3.3910255432128906,
      "step": 85
    },
    {
      "epoch": 0.19235907026449373,
      "grad_norm": 7.509298977705249,
      "learning_rate": 4.871796225971999e-07,
      "logits/chosen": -4.371309280395508,
      "logits/rejected": -4.230607509613037,
      "logps/chosen": -1.4209774732589722,
      "logps/rejected": -1.3977888822555542,
      "loss": 1.7377,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -3.552443265914917,
      "rewards/margins": -0.05797123908996582,
      "rewards/rejected": -3.494471788406372,
      "step": 90
    },
    {
      "epoch": 0.20304568527918782,
      "grad_norm": 7.494464457003994,
      "learning_rate": 4.840587176599343e-07,
      "logits/chosen": -4.4767680168151855,
      "logits/rejected": -4.559601306915283,
      "logps/chosen": -1.3669825792312622,
      "logps/rejected": -1.4395772218704224,
      "loss": 1.7103,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -3.4174563884735107,
      "rewards/margins": 0.181486576795578,
      "rewards/rejected": -3.598942995071411,
      "step": 95
    },
    {
      "epoch": 0.2137323002938819,
      "grad_norm": 9.417753270846694,
      "learning_rate": 4.806104604416823e-07,
      "logits/chosen": -4.715841770172119,
      "logits/rejected": -4.59860897064209,
      "logps/chosen": -1.341230869293213,
      "logps/rejected": -1.2855507135391235,
      "loss": 1.7501,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -3.3530774116516113,
      "rewards/margins": -0.13920024037361145,
      "rewards/rejected": -3.2138772010803223,
      "step": 100
    },
    {
      "epoch": 0.224418915308576,
      "grad_norm": 8.25939695647074,
      "learning_rate": 4.768396736419662e-07,
      "logits/chosen": -4.579805850982666,
      "logits/rejected": -4.472721576690674,
      "logps/chosen": -1.2403936386108398,
      "logps/rejected": -1.2722570896148682,
      "loss": 1.7103,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -3.1009843349456787,
      "rewards/margins": 0.07965835183858871,
      "rewards/rejected": -3.180642604827881,
      "step": 105
    },
    {
      "epoch": 0.2351055303232701,
      "grad_norm": 7.0286538282806905,
      "learning_rate": 4.7275163104709194e-07,
      "logits/chosen": -4.310456275939941,
      "logits/rejected": -4.180710315704346,
      "logps/chosen": -1.3125991821289062,
      "logps/rejected": -1.3054355382919312,
      "loss": 1.7814,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -3.2814979553222656,
      "rewards/margins": -0.017909258604049683,
      "rewards/rejected": -3.2635891437530518,
      "step": 110
    },
    {
      "epoch": 0.2457921453379642,
      "grad_norm": 7.4251876749652315,
      "learning_rate": 4.683520501542824e-07,
      "logits/chosen": -4.345327854156494,
      "logits/rejected": -4.324080944061279,
      "logps/chosen": -1.269272804260254,
      "logps/rejected": -1.255380392074585,
      "loss": 1.7077,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -3.1731820106506348,
      "rewards/margins": -0.03473084047436714,
      "rewards/rejected": -3.138451099395752,
      "step": 115
    },
    {
      "epoch": 0.2564787603526583,
      "grad_norm": 7.68817663348518,
      "learning_rate": 4.636470841752404e-07,
      "logits/chosen": -4.619086265563965,
      "logits/rejected": -4.4949049949646,
      "logps/chosen": -1.2920095920562744,
      "logps/rejected": -1.2791041135787964,
      "loss": 1.7086,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -3.2300238609313965,
      "rewards/margins": -0.03226359933614731,
      "rewards/rejected": -3.1977603435516357,
      "step": 120
    },
    {
      "epoch": 0.2671653753673524,
      "grad_norm": 7.125331982786664,
      "learning_rate": 4.5864331343032565e-07,
      "logits/chosen": -4.59976053237915,
      "logits/rejected": -4.594804286956787,
      "logps/chosen": -1.3433685302734375,
      "logps/rejected": -1.4002816677093506,
      "loss": 1.729,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -3.3584213256835938,
      "rewards/margins": 0.14228278398513794,
      "rewards/rejected": -3.500704288482666,
      "step": 125
    },
    {
      "epoch": 0.2778519903820465,
      "grad_norm": 8.312615727941836,
      "learning_rate": 4.533477361453819e-07,
      "logits/chosen": -4.355613708496094,
      "logits/rejected": -4.472823619842529,
      "logps/chosen": -1.3405156135559082,
      "logps/rejected": -1.3726789951324463,
      "loss": 1.7552,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -3.3512885570526123,
      "rewards/margins": 0.08040888607501984,
      "rewards/rejected": -3.431697368621826,
      "step": 130
    },
    {
      "epoch": 0.2885386053967406,
      "grad_norm": 8.624812004095546,
      "learning_rate": 4.4776775866408533e-07,
      "logits/chosen": -4.605002403259277,
      "logits/rejected": -4.486802577972412,
      "logps/chosen": -1.2775276899337769,
      "logps/rejected": -1.2938398122787476,
      "loss": 1.7193,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -3.193819284439087,
      "rewards/margins": 0.040780358016490936,
      "rewards/rejected": -3.2345993518829346,
      "step": 135
    },
    {
      "epoch": 0.2992252204114347,
      "grad_norm": 8.928849598835027,
      "learning_rate": 4.4191118508950277e-07,
      "logits/chosen": -4.492905616760254,
      "logits/rejected": -4.466243743896484,
      "logps/chosen": -1.4048876762390137,
      "logps/rejected": -1.4387624263763428,
      "loss": 1.7204,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -3.512219190597534,
      "rewards/margins": 0.08468694984912872,
      "rewards/rejected": -3.5969061851501465,
      "step": 140
    },
    {
      "epoch": 0.30991183542612877,
      "grad_norm": 8.071911039943062,
      "learning_rate": 4.357862063693485e-07,
      "logits/chosen": -4.527454376220703,
      "logits/rejected": -4.5367279052734375,
      "logps/chosen": -1.344947099685669,
      "logps/rejected": -1.371901273727417,
      "loss": 1.7418,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -3.362368106842041,
      "rewards/margins": 0.06738501042127609,
      "rewards/rejected": -3.429752826690674,
      "step": 145
    },
    {
      "epoch": 0.32059845044082286,
      "grad_norm": 8.671271998589708,
      "learning_rate": 4.294013888402029e-07,
      "logits/chosen": -4.329895496368408,
      "logits/rejected": -4.347752571105957,
      "logps/chosen": -1.2269880771636963,
      "logps/rejected": -1.3332128524780273,
      "loss": 1.7002,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -3.0674703121185303,
      "rewards/margins": 0.2655617594718933,
      "rewards/rejected": -3.3330321311950684,
      "step": 150
    },
    {
      "epoch": 0.33128506545551695,
      "grad_norm": 11.410050720477157,
      "learning_rate": 4.227656622467162e-07,
      "logits/chosen": -4.60593843460083,
      "logits/rejected": -4.5206708908081055,
      "logps/chosen": -1.2302855253219604,
      "logps/rejected": -1.285362958908081,
      "loss": 1.7276,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -3.075713872909546,
      "rewards/margins": 0.13769367337226868,
      "rewards/rejected": -3.2134079933166504,
      "step": 155
    },
    {
      "epoch": 0.34197168047021104,
      "grad_norm": 8.929425526594324,
      "learning_rate": 4.158883072525528e-07,
      "logits/chosen": -4.636877536773682,
      "logits/rejected": -4.64215612411499,
      "logps/chosen": -1.2857083082199097,
      "logps/rejected": -1.2909152507781982,
      "loss": 1.7261,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -3.214270830154419,
      "rewards/margins": 0.01301727257668972,
      "rewards/rejected": -3.227288007736206,
      "step": 160
    },
    {
      "epoch": 0.3526582954849052,
      "grad_norm": 8.999759770579269,
      "learning_rate": 4.087789424605447e-07,
      "logits/chosen": -4.477012634277344,
      "logits/rejected": -4.342657089233398,
      "logps/chosen": -1.2676000595092773,
      "logps/rejected": -1.2652432918548584,
      "loss": 1.7307,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -3.168999671936035,
      "rewards/margins": -0.005891731474548578,
      "rewards/rejected": -3.1631083488464355,
      "step": 165
    },
    {
      "epoch": 0.36334491049959927,
      "grad_norm": 7.38370964011058,
      "learning_rate": 4.0144751096020497e-07,
      "logits/chosen": -4.322202682495117,
      "logits/rejected": -4.344858646392822,
      "logps/chosen": -1.3334827423095703,
      "logps/rejected": -1.3564600944519043,
      "loss": 1.7042,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -3.3337066173553467,
      "rewards/margins": 0.05744355916976929,
      "rewards/rejected": -3.3911502361297607,
      "step": 170
    },
    {
      "epoch": 0.37403152551429336,
      "grad_norm": 7.609720560260437,
      "learning_rate": 3.939042664214184e-07,
      "logits/chosen": -4.588566780090332,
      "logits/rejected": -4.577520847320557,
      "logps/chosen": -1.3666011095046997,
      "logps/rejected": -1.4189504384994507,
      "loss": 1.6971,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -3.4165027141571045,
      "rewards/margins": 0.13087329268455505,
      "rewards/rejected": -3.5473761558532715,
      "step": 175
    },
    {
      "epoch": 0.38471814052898745,
      "grad_norm": 7.522883946304073,
      "learning_rate": 3.8615975875375676e-07,
      "logits/chosen": -4.611153602600098,
      "logits/rejected": -4.639096736907959,
      "logps/chosen": -1.3451780080795288,
      "logps/rejected": -1.407920241355896,
      "loss": 1.7286,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -3.3629448413848877,
      "rewards/margins": 0.15685562789440155,
      "rewards/rejected": -3.5198006629943848,
      "step": 180
    },
    {
      "epoch": 0.39540475554368154,
      "grad_norm": 8.15961865987409,
      "learning_rate": 3.7822481935147655e-07,
      "logits/chosen": -4.689408302307129,
      "logits/rejected": -4.5546979904174805,
      "logps/chosen": -1.3114018440246582,
      "logps/rejected": -1.333992838859558,
      "loss": 1.7275,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -3.2785046100616455,
      "rewards/margins": 0.05647749826312065,
      "rewards/rejected": -3.334981918334961,
      "step": 185
    },
    {
      "epoch": 0.40609137055837563,
      "grad_norm": 8.427441389080544,
      "learning_rate": 3.7011054594483443e-07,
      "logits/chosen": -4.588101863861084,
      "logits/rejected": -4.52250862121582,
      "logps/chosen": -1.3444761037826538,
      "logps/rejected": -1.4100086688995361,
      "loss": 1.6814,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -3.3611907958984375,
      "rewards/margins": 0.16383083164691925,
      "rewards/rejected": -3.5250213146209717,
      "step": 190
    },
    {
      "epoch": 0.4167779855730697,
      "grad_norm": 8.693634103691119,
      "learning_rate": 3.618282870789081e-07,
      "logits/chosen": -4.41694450378418,
      "logits/rejected": -4.407210826873779,
      "logps/chosen": -1.3954448699951172,
      "logps/rejected": -1.3883302211761475,
      "loss": 1.7438,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -3.488612413406372,
      "rewards/margins": -0.017786988988518715,
      "rewards/rejected": -3.4708251953125,
      "step": 195
    },
    {
      "epoch": 0.4274646005877638,
      "grad_norm": 7.817917194646963,
      "learning_rate": 3.5338962624163016e-07,
      "logits/chosen": -4.41127872467041,
      "logits/rejected": -4.464591026306152,
      "logps/chosen": -1.3187341690063477,
      "logps/rejected": -1.3379795551300049,
      "loss": 1.7493,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -3.296835422515869,
      "rewards/margins": 0.04811403155326843,
      "rewards/rejected": -3.344949245452881,
      "step": 200
    },
    {
      "epoch": 0.4381512156024579,
      "grad_norm": 7.87816681095113,
      "learning_rate": 3.448063656632321e-07,
      "logits/chosen": -4.585541248321533,
      "logits/rejected": -4.440451145172119,
      "logps/chosen": -1.3579752445220947,
      "logps/rejected": -1.3495149612426758,
      "loss": 1.6908,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -3.3949382305145264,
      "rewards/margins": -0.02115105651319027,
      "rewards/rejected": -3.3737874031066895,
      "step": 205
    },
    {
      "epoch": 0.448837830617152,
      "grad_norm": 8.192581867844957,
      "learning_rate": 3.360905098097587e-07,
      "logits/chosen": -4.702603340148926,
      "logits/rejected": -4.5605058670043945,
      "logps/chosen": -1.3157254457473755,
      "logps/rejected": -1.326529860496521,
      "loss": 1.7765,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -3.289313554763794,
      "rewards/margins": 0.02701149880886078,
      "rewards/rejected": -3.3163249492645264,
      "step": 210
    },
    {
      "epoch": 0.45952444563184613,
      "grad_norm": 9.916408251919977,
      "learning_rate": 3.272542485937368e-07,
      "logits/chosen": -4.547693252563477,
      "logits/rejected": -4.488272666931152,
      "logps/chosen": -1.2768534421920776,
      "logps/rejected": -1.3246023654937744,
      "loss": 1.6999,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -3.192133665084839,
      "rewards/margins": 0.11937196552753448,
      "rewards/rejected": -3.3115055561065674,
      "step": 215
    },
    {
      "epoch": 0.4702110606465402,
      "grad_norm": 11.54297063214211,
      "learning_rate": 3.1830994032548e-07,
      "logits/chosen": -4.549879550933838,
      "logits/rejected": -4.424410820007324,
      "logps/chosen": -1.354941725730896,
      "logps/rejected": -1.3328911066055298,
      "loss": 1.7459,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -3.387354612350464,
      "rewards/margins": -0.05512649938464165,
      "rewards/rejected": -3.332228183746338,
      "step": 220
    },
    {
      "epoch": 0.4808976756612343,
      "grad_norm": 7.745936948551203,
      "learning_rate": 3.0927009442887437e-07,
      "logits/chosen": -4.522200584411621,
      "logits/rejected": -4.433478355407715,
      "logps/chosen": -1.3177942037582397,
      "logps/rejected": -1.312595248222351,
      "loss": 1.7151,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -3.294485569000244,
      "rewards/margins": -0.012997297570109367,
      "rewards/rejected": -3.2814879417419434,
      "step": 225
    },
    {
      "epoch": 0.4915842906759284,
      "grad_norm": 8.982993035215612,
      "learning_rate": 3.001473539458182e-07,
      "logits/chosen": -4.790830135345459,
      "logits/rejected": -4.502416610717773,
      "logps/chosen": -1.2848570346832275,
      "logps/rejected": -1.3198983669281006,
      "loss": 1.7007,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -3.2121429443359375,
      "rewards/margins": 0.08760317414999008,
      "rewards/rejected": -3.299746036529541,
      "step": 230
    },
    {
      "epoch": 0.5022709056906225,
      "grad_norm": 7.9015489492124225,
      "learning_rate": 2.909544778537844e-07,
      "logits/chosen": -4.630471229553223,
      "logits/rejected": -4.5115509033203125,
      "logps/chosen": -1.3522698879241943,
      "logps/rejected": -1.337274193763733,
      "loss": 1.7185,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": -3.3806750774383545,
      "rewards/margins": -0.03748970106244087,
      "rewards/rejected": -3.3431854248046875,
      "step": 235
    },
    {
      "epoch": 0.5129575207053166,
      "grad_norm": 8.651538383776115,
      "learning_rate": 2.817043232212371e-07,
      "logits/chosen": -4.58451509475708,
      "logits/rejected": -4.510278701782227,
      "logps/chosen": -1.3096239566802979,
      "logps/rejected": -1.3383967876434326,
      "loss": 1.6684,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -3.274059295654297,
      "rewards/margins": 0.07193277776241302,
      "rewards/rejected": -3.34599232673645,
      "step": 240
    },
    {
      "epoch": 0.5236441357200107,
      "grad_norm": 8.276564878362981,
      "learning_rate": 2.7240982722585837e-07,
      "logits/chosen": -4.700469970703125,
      "logits/rejected": -4.604641914367676,
      "logps/chosen": -1.296589970588684,
      "logps/rejected": -1.295531988143921,
      "loss": 1.7276,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -3.2414746284484863,
      "rewards/margins": -0.0026448846329003572,
      "rewards/rejected": -3.2388298511505127,
      "step": 245
    },
    {
      "epoch": 0.5343307507347048,
      "grad_norm": 8.739959615068186,
      "learning_rate": 2.63083989060736e-07,
      "logits/chosen": -4.688320636749268,
      "logits/rejected": -4.622137069702148,
      "logps/chosen": -1.420803427696228,
      "logps/rejected": -1.5039843320846558,
      "loss": 1.706,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -3.552008867263794,
      "rewards/margins": 0.20795249938964844,
      "rewards/rejected": -3.7599613666534424,
      "step": 250
    },
    {
      "epoch": 0.5450173657493989,
      "grad_norm": 8.205463032414816,
      "learning_rate": 2.537398517538159e-07,
      "logits/chosen": -4.498805046081543,
      "logits/rejected": -4.545159816741943,
      "logps/chosen": -1.3838533163070679,
      "logps/rejected": -1.427234411239624,
      "loss": 1.7113,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -3.4596335887908936,
      "rewards/margins": 0.10845281928777695,
      "rewards/rejected": -3.5680861473083496,
      "step": 255
    },
    {
      "epoch": 0.555703980764093,
      "grad_norm": 9.403793469995996,
      "learning_rate": 2.4439048392604877e-07,
      "logits/chosen": -4.465289115905762,
      "logits/rejected": -4.415510654449463,
      "logps/chosen": -1.3352388143539429,
      "logps/rejected": -1.4621715545654297,
      "loss": 1.6556,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -3.338097095489502,
      "rewards/margins": 0.3173316717147827,
      "rewards/rejected": -3.655428647994995,
      "step": 260
    },
    {
      "epoch": 0.566390595778787,
      "grad_norm": 9.87099892276829,
      "learning_rate": 2.3504896151374144e-07,
      "logits/chosen": -4.503221035003662,
      "logits/rejected": -4.532763481140137,
      "logps/chosen": -1.3822702169418335,
      "logps/rejected": -1.3944904804229736,
      "loss": 1.7068,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -3.4556756019592285,
      "rewards/margins": 0.03055066429078579,
      "rewards/rejected": -3.4862258434295654,
      "step": 265
    },
    {
      "epoch": 0.5770772107934812,
      "grad_norm": 9.709835641918701,
      "learning_rate": 2.2572834948067795e-07,
      "logits/chosen": -4.5645036697387695,
      "logits/rejected": -4.5933027267456055,
      "logps/chosen": -1.2755674123764038,
      "logps/rejected": -1.3178540468215942,
      "loss": 1.6978,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -3.1889185905456543,
      "rewards/margins": 0.10571658611297607,
      "rewards/rejected": -3.294635057449341,
      "step": 270
    },
    {
      "epoch": 0.5877638258081752,
      "grad_norm": 10.023847803695947,
      "learning_rate": 2.164416835455862e-07,
      "logits/chosen": -4.247835159301758,
      "logits/rejected": -4.194937229156494,
      "logps/chosen": -1.3332767486572266,
      "logps/rejected": -1.3482983112335205,
      "loss": 1.6989,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -3.3331921100616455,
      "rewards/margins": 0.03755370154976845,
      "rewards/rejected": -3.3707454204559326,
      "step": 275
    },
    {
      "epoch": 0.5984504408228694,
      "grad_norm": 12.08817520180493,
      "learning_rate": 2.072019519505062e-07,
      "logits/chosen": -4.784656524658203,
      "logits/rejected": -4.656561851501465,
      "logps/chosen": -1.3093583583831787,
      "logps/rejected": -1.3450324535369873,
      "loss": 1.6805,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -3.2733960151672363,
      "rewards/margins": 0.08918508142232895,
      "rewards/rejected": -3.362581253051758,
      "step": 280
    },
    {
      "epoch": 0.6091370558375635,
      "grad_norm": 8.794437977214896,
      "learning_rate": 1.980220772955602e-07,
      "logits/chosen": -4.4688849449157715,
      "logits/rejected": -4.370944499969482,
      "logps/chosen": -1.335540533065796,
      "logps/rejected": -1.3469078540802002,
      "loss": 1.6982,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -3.3388514518737793,
      "rewards/margins": 0.028418350964784622,
      "rewards/rejected": -3.367269992828369,
      "step": 285
    },
    {
      "epoch": 0.6198236708522575,
      "grad_norm": 13.055246394354226,
      "learning_rate": 1.8891489846552644e-07,
      "logits/chosen": -4.446020603179932,
      "logits/rejected": -4.3907389640808105,
      "logps/chosen": -1.332921028137207,
      "logps/rejected": -1.3605272769927979,
      "loss": 1.646,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -3.3323025703430176,
      "rewards/margins": 0.06901548057794571,
      "rewards/rejected": -3.401318311691284,
      "step": 290
    },
    {
      "epoch": 0.6305102858669517,
      "grad_norm": 9.19761267983714,
      "learning_rate": 1.7989315267349933e-07,
      "logits/chosen": -4.640501022338867,
      "logits/rejected": -4.582611083984375,
      "logps/chosen": -1.345268964767456,
      "logps/rejected": -1.3664015531539917,
      "loss": 1.6828,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -3.363173007965088,
      "rewards/margins": 0.05283108353614807,
      "rewards/rejected": -3.416003704071045,
      "step": 295
    },
    {
      "epoch": 0.6411969008816457,
      "grad_norm": 8.163371854491718,
      "learning_rate": 1.7096945764674398e-07,
      "logits/chosen": -4.53090763092041,
      "logits/rejected": -4.449822425842285,
      "logps/chosen": -1.357412338256836,
      "logps/rejected": -1.340632677078247,
      "loss": 1.7361,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -3.393531084060669,
      "rewards/margins": -0.04194915294647217,
      "rewards/rejected": -3.3515820503234863,
      "step": 300
    },
    {
      "epoch": 0.6518835158963399,
      "grad_norm": 9.037247969057203,
      "learning_rate": 1.621562939796643e-07,
      "logits/chosen": -4.495213508605957,
      "logits/rejected": -4.3966827392578125,
      "logps/chosen": -1.4379794597625732,
      "logps/rejected": -1.473179578781128,
      "loss": 1.7018,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -3.5949490070343018,
      "rewards/margins": 0.08800011873245239,
      "rewards/rejected": -3.682948589324951,
      "step": 305
    },
    {
      "epoch": 0.6625701309110339,
      "grad_norm": 13.030910514110566,
      "learning_rate": 1.5346598767856345e-07,
      "logits/chosen": -4.379870414733887,
      "logits/rejected": -4.560307025909424,
      "logps/chosen": -1.398297905921936,
      "logps/rejected": -1.4838316440582275,
      "loss": 1.6849,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -3.4957451820373535,
      "rewards/margins": 0.21383419632911682,
      "rewards/rejected": -3.7095787525177,
      "step": 310
    },
    {
      "epoch": 0.673256745925728,
      "grad_norm": 9.601996227444385,
      "learning_rate": 1.4491069292260866e-07,
      "logits/chosen": -4.930965423583984,
      "logits/rejected": -4.876108646392822,
      "logps/chosen": -1.430011510848999,
      "logps/rejected": -1.4526395797729492,
      "loss": 1.7084,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -3.575028896331787,
      "rewards/margins": 0.05657036229968071,
      "rewards/rejected": -3.631598949432373,
      "step": 315
    },
    {
      "epoch": 0.6839433609404221,
      "grad_norm": 8.370911393375984,
      "learning_rate": 1.365023750651133e-07,
      "logits/chosen": -4.7936224937438965,
      "logits/rejected": -4.740353584289551,
      "logps/chosen": -1.282147765159607,
      "logps/rejected": -1.4656821489334106,
      "loss": 1.6538,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -3.205369472503662,
      "rewards/margins": 0.4588359296321869,
      "rewards/rejected": -3.6642050743103027,
      "step": 320
    },
    {
      "epoch": 0.6946299759551162,
      "grad_norm": 10.059762786625233,
      "learning_rate": 1.2825279389890818e-07,
      "logits/chosen": -4.382018089294434,
      "logits/rejected": -4.410761833190918,
      "logps/chosen": -1.3325350284576416,
      "logps/rejected": -1.4160031080245972,
      "loss": 1.6493,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -3.3313376903533936,
      "rewards/margins": 0.20867028832435608,
      "rewards/rejected": -3.5400078296661377,
      "step": 325
    },
    {
      "epoch": 0.7053165909698104,
      "grad_norm": 9.161029493778742,
      "learning_rate": 1.201734872092077e-07,
      "logits/chosen": -4.73400354385376,
      "logits/rejected": -4.58245849609375,
      "logps/chosen": -1.3593249320983887,
      "logps/rejected": -1.5017703771591187,
      "loss": 1.6762,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -3.3983123302459717,
      "rewards/margins": 0.3561137616634369,
      "rewards/rejected": -3.7544262409210205,
      "step": 330
    },
    {
      "epoch": 0.7160032059845044,
      "grad_norm": 8.300761837819268,
      "learning_rate": 1.1227575463697439e-07,
      "logits/chosen": -4.566507816314697,
      "logits/rejected": -4.648960590362549,
      "logps/chosen": -1.3698114156723022,
      "logps/rejected": -1.4295297861099243,
      "loss": 1.683,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -3.4245285987854004,
      "rewards/margins": 0.14929580688476562,
      "rewards/rejected": -3.573824405670166,
      "step": 335
    },
    {
      "epoch": 0.7266898209991985,
      "grad_norm": 9.524750095557723,
      "learning_rate": 1.0457064187534861e-07,
      "logits/chosen": -4.421561241149902,
      "logits/rejected": -4.395345687866211,
      "logps/chosen": -1.2686647176742554,
      "logps/rejected": -1.3926507234573364,
      "loss": 1.7023,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -3.171661615371704,
      "rewards/margins": 0.3099651038646698,
      "rewards/rejected": -3.4816269874572754,
      "step": 340
    },
    {
      "epoch": 0.7373764360138926,
      "grad_norm": 9.264218645867375,
      "learning_rate": 9.706892522124838e-08,
      "logits/chosen": -4.445697784423828,
      "logits/rejected": -4.289374351501465,
      "logps/chosen": -1.3950523138046265,
      "logps/rejected": -1.3890727758407593,
      "loss": 1.6517,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -3.487631320953369,
      "rewards/margins": -0.014949033036828041,
      "rewards/rejected": -3.472681760787964,
      "step": 345
    },
    {
      "epoch": 0.7480630510285867,
      "grad_norm": 9.805135028816036,
      "learning_rate": 8.978109650374396e-08,
      "logits/chosen": -4.54650354385376,
      "logits/rejected": -4.534266471862793,
      "logps/chosen": -1.4010884761810303,
      "logps/rejected": -1.4561593532562256,
      "loss": 1.7054,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -3.5027213096618652,
      "rewards/margins": 0.13767699897289276,
      "rewards/rejected": -3.6403985023498535,
      "step": 350
    },
    {
      "epoch": 0.7587496660432808,
      "grad_norm": 13.674204223054398,
      "learning_rate": 8.271734841028552e-08,
      "logits/chosen": -4.590233325958252,
      "logits/rejected": -4.405337333679199,
      "logps/chosen": -1.3775599002838135,
      "logps/rejected": -1.3799619674682617,
      "loss": 1.6717,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": -3.4438998699188232,
      "rewards/margins": 0.0060052452608942986,
      "rewards/rejected": -3.4499053955078125,
      "step": 355
    },
    {
      "epoch": 0.7694362810579749,
      "grad_norm": 8.839622500870057,
      "learning_rate": 7.588756023130833e-08,
      "logits/chosen": -4.494322776794434,
      "logits/rejected": -4.6520586013793945,
      "logps/chosen": -1.43705153465271,
      "logps/rejected": -1.5561714172363281,
      "loss": 1.6713,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -3.5926284790039062,
      "rewards/margins": 0.2977997660636902,
      "rewards/rejected": -3.890428066253662,
      "step": 360
    },
    {
      "epoch": 0.7801228960726689,
      "grad_norm": 7.5612671070202895,
      "learning_rate": 6.930128404315214e-08,
      "logits/chosen": -4.652656555175781,
      "logits/rejected": -4.5095977783203125,
      "logps/chosen": -1.4330899715423584,
      "logps/rejected": -1.393972635269165,
      "loss": 1.6884,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": -3.5827248096466064,
      "rewards/margins": -0.09779311716556549,
      "rewards/rejected": -3.484931468963623,
      "step": 365
    },
    {
      "epoch": 0.7908095110873631,
      "grad_norm": 10.689889310789306,
      "learning_rate": 6.296773134861824e-08,
      "logits/chosen": -4.595254421234131,
      "logits/rejected": -4.460488796234131,
      "logps/chosen": -1.455096960067749,
      "logps/rejected": -1.4411834478378296,
      "loss": 1.7402,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -3.637742280960083,
      "rewards/margins": -0.034783538430929184,
      "rewards/rejected": -3.602959156036377,
      "step": 370
    },
    {
      "epoch": 0.8014961261020572,
      "grad_norm": 11.234917641139386,
      "learning_rate": 5.6895760193850145e-08,
      "logits/chosen": -4.608285903930664,
      "logits/rejected": -4.652877330780029,
      "logps/chosen": -1.3920633792877197,
      "logps/rejected": -1.4802567958831787,
      "loss": 1.6655,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -3.480158567428589,
      "rewards/margins": 0.22048339247703552,
      "rewards/rejected": -3.7006423473358154,
      "step": 375
    },
    {
      "epoch": 0.8121827411167513,
      "grad_norm": 9.88912726576546,
      "learning_rate": 5.109386277955477e-08,
      "logits/chosen": -4.704692840576172,
      "logits/rejected": -4.580458641052246,
      "logps/chosen": -1.4294003248214722,
      "logps/rejected": -1.4911302328109741,
      "loss": 1.6673,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -3.573500871658325,
      "rewards/margins": 0.15432511270046234,
      "rewards/rejected": -3.72782564163208,
      "step": 380
    },
    {
      "epoch": 0.8228693561314454,
      "grad_norm": 10.411305158642252,
      "learning_rate": 4.557015358389216e-08,
      "logits/chosen": -4.629119396209717,
      "logits/rejected": -4.591732978820801,
      "logps/chosen": -1.371460199356079,
      "logps/rejected": -1.4805608987808228,
      "loss": 1.6999,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -3.4286506175994873,
      "rewards/margins": 0.27275168895721436,
      "rewards/rejected": -3.701402187347412,
      "step": 385
    },
    {
      "epoch": 0.8335559711461394,
      "grad_norm": 10.400854396409033,
      "learning_rate": 4.0332358013644015e-08,
      "logits/chosen": -4.841821670532227,
      "logits/rejected": -4.738966941833496,
      "logps/chosen": -1.4363723993301392,
      "logps/rejected": -1.4949367046356201,
      "loss": 1.6777,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -3.590930938720703,
      "rewards/margins": 0.14641080796718597,
      "rewards/rejected": -3.7373416423797607,
      "step": 390
    },
    {
      "epoch": 0.8442425861608336,
      "grad_norm": 7.772358738977712,
      "learning_rate": 3.538780159953347e-08,
      "logits/chosen": -4.330094337463379,
      "logits/rejected": -4.12509298324585,
      "logps/chosen": -1.3891313076019287,
      "logps/rejected": -1.3778597116470337,
      "loss": 1.7104,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": -3.4728283882141113,
      "rewards/margins": -0.02817906066775322,
      "rewards/rejected": -3.4446492195129395,
      "step": 395
    },
    {
      "epoch": 0.8549292011755276,
      "grad_norm": 8.138330588799487,
      "learning_rate": 3.074339975080836e-08,
      "logits/chosen": -4.4906816482543945,
      "logits/rejected": -4.360891819000244,
      "logps/chosen": -1.4243663549423218,
      "logps/rejected": -1.5000258684158325,
      "loss": 1.6693,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -3.5609164237976074,
      "rewards/margins": 0.18914875388145447,
      "rewards/rejected": -3.7500648498535156,
      "step": 400
    },
    {
      "epoch": 0.8549292011755276,
      "eval_logits/chosen": -5.357321739196777,
      "eval_logits/rejected": -5.259011745452881,
      "eval_logps/chosen": -1.3368273973464966,
      "eval_logps/rejected": -1.3893804550170898,
      "eval_loss": 1.6597751379013062,
      "eval_rewards/accuracies": 0.5282257795333862,
      "eval_rewards/chosen": -3.342068672180176,
      "eval_rewards/margins": 0.13138233125209808,
      "eval_rewards/rejected": -3.4734511375427246,
      "eval_runtime": 31.809,
      "eval_samples_per_second": 61.649,
      "eval_steps_per_second": 1.949,
      "step": 400
    },
    {
      "epoch": 0.8656158161902218,
      "grad_norm": 9.025721024350105,
      "learning_rate": 2.6405648083415833e-08,
      "logits/chosen": -4.550287246704102,
      "logits/rejected": -4.305168628692627,
      "logps/chosen": -1.3779646158218384,
      "logps/rejected": -1.3646600246429443,
      "loss": 1.7228,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -3.444911479949951,
      "rewards/margins": -0.033261336386203766,
      "rewards/rejected": -3.4116501808166504,
      "step": 405
    },
    {
      "epoch": 0.8763024312049158,
      "grad_norm": 11.89923381726874,
      "learning_rate": 2.2380613335296033e-08,
      "logits/chosen": -4.46389102935791,
      "logits/rejected": -4.445181846618652,
      "logps/chosen": -1.3117091655731201,
      "logps/rejected": -1.3666512966156006,
      "loss": 1.6743,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -3.2792727947235107,
      "rewards/margins": 0.13735508918762207,
      "rewards/rejected": -3.416628360748291,
      "step": 410
    },
    {
      "epoch": 0.88698904621961,
      "grad_norm": 9.427141264972793,
      "learning_rate": 1.8673924881500823e-08,
      "logits/chosen": -4.327395439147949,
      "logits/rejected": -4.3094940185546875,
      "logps/chosen": -1.3074382543563843,
      "logps/rejected": -1.3440717458724976,
      "loss": 1.6969,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -3.2685959339141846,
      "rewards/margins": 0.09158400446176529,
      "rewards/rejected": -3.3601794242858887,
      "step": 415
    },
    {
      "epoch": 0.897675661234304,
      "grad_norm": 8.90218297589139,
      "learning_rate": 1.5290766861003475e-08,
      "logits/chosen": -4.604608058929443,
      "logits/rejected": -4.5904130935668945,
      "logps/chosen": -1.3577730655670166,
      "logps/rejected": -1.4380416870117188,
      "loss": 1.6707,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -3.394432544708252,
      "rewards/margins": 0.20067182183265686,
      "rewards/rejected": -3.595104694366455,
      "step": 420
    },
    {
      "epoch": 0.9083622762489981,
      "grad_norm": 10.88127447829711,
      "learning_rate": 1.2235870926211616e-08,
      "logits/chosen": -4.505583763122559,
      "logits/rejected": -4.377171516418457,
      "logps/chosen": -1.379010558128357,
      "logps/rejected": -1.4630334377288818,
      "loss": 1.6586,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -3.447526454925537,
      "rewards/margins": 0.21005713939666748,
      "rewards/rejected": -3.6575839519500732,
      "step": 425
    },
    {
      "epoch": 0.9190488912636923,
      "grad_norm": 10.139040628163542,
      "learning_rate": 9.513509625323518e-09,
      "logits/chosen": -4.474516868591309,
      "logits/rejected": -4.397044658660889,
      "logps/chosen": -1.3109729290008545,
      "logps/rejected": -1.3905436992645264,
      "loss": 1.6431,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -3.2774322032928467,
      "rewards/margins": 0.19892734289169312,
      "rewards/rejected": -3.4763596057891846,
      "step": 430
    },
    {
      "epoch": 0.9297355062783863,
      "grad_norm": 9.603243560454548,
      "learning_rate": 7.127490426783123e-09,
      "logits/chosen": -4.4349188804626465,
      "logits/rejected": -4.401902675628662,
      "logps/chosen": -1.382652997970581,
      "logps/rejected": -1.4352567195892334,
      "loss": 1.6374,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -3.456632137298584,
      "rewards/margins": 0.13150940835475922,
      "rewards/rejected": -3.588141918182373,
      "step": 435
    },
    {
      "epoch": 0.9404221212930804,
      "grad_norm": 9.022313878003665,
      "learning_rate": 5.08115039419113e-09,
      "logits/chosen": -4.515431880950928,
      "logits/rejected": -4.496222496032715,
      "logps/chosen": -1.3486416339874268,
      "logps/rejected": -1.328399896621704,
      "loss": 1.6747,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -3.3716042041778564,
      "rewards/margins": -0.05060465261340141,
      "rewards/rejected": -3.3209996223449707,
      "step": 440
    },
    {
      "epoch": 0.9511087363077745,
      "grad_norm": 10.793631217244918,
      "learning_rate": 3.3773515191196646e-09,
      "logits/chosen": -4.495880126953125,
      "logits/rejected": -4.391939640045166,
      "logps/chosen": -1.355960726737976,
      "logps/rejected": -1.3439215421676636,
      "loss": 1.6919,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -3.389902114868164,
      "rewards/margins": -0.03009791299700737,
      "rewards/rejected": -3.3598036766052246,
      "step": 445
    },
    {
      "epoch": 0.9617953513224686,
      "grad_norm": 11.320376659575638,
      "learning_rate": 2.0184767183584474e-09,
      "logits/chosen": -4.227232456207275,
      "logits/rejected": -4.283637523651123,
      "logps/chosen": -1.4779750108718872,
      "logps/rejected": -1.5757644176483154,
      "loss": 1.6992,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -3.694937229156494,
      "rewards/margins": 0.24447353184223175,
      "rewards/rejected": -3.939411163330078,
      "step": 450
    },
    {
      "epoch": 0.9724819663371627,
      "grad_norm": 9.391067862633497,
      "learning_rate": 1.0064265011902328e-09,
      "logits/chosen": -4.655524253845215,
      "logits/rejected": -4.6156415939331055,
      "logps/chosen": -1.4518479108810425,
      "logps/rejected": -1.3639124631881714,
      "loss": 1.7101,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -3.62962007522583,
      "rewards/margins": -0.21983873844146729,
      "rewards/rejected": -3.4097812175750732,
      "step": 455
    },
    {
      "epoch": 0.9831685813518568,
      "grad_norm": 9.291539758676725,
      "learning_rate": 3.4261631135654167e-10,
      "logits/chosen": -4.43479585647583,
      "logits/rejected": -4.606427192687988,
      "logps/chosen": -1.3911080360412598,
      "logps/rejected": -1.5079371929168701,
      "loss": 1.6583,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -3.4777698516845703,
      "rewards/margins": 0.29207324981689453,
      "rewards/rejected": -3.769843339920044,
      "step": 460
    },
    {
      "epoch": 0.9938551963665508,
      "grad_norm": 9.789963041329289,
      "learning_rate": 2.797454743164174e-11,
      "logits/chosen": -4.726589679718018,
      "logits/rejected": -4.504673957824707,
      "logps/chosen": -1.3526123762130737,
      "logps/rejected": -1.3742250204086304,
      "loss": 1.6691,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -3.38153076171875,
      "rewards/margins": 0.05403166264295578,
      "rewards/rejected": -3.4355628490448,
      "step": 465
    },
    {
      "epoch": 0.9981298423724285,
      "step": 467,
      "total_flos": 0.0,
      "train_loss": 1.7099600828689476,
      "train_runtime": 3104.7014,
      "train_samples_per_second": 19.286,
      "train_steps_per_second": 0.15
    }
  ],
  "logging_steps": 5,
  "max_steps": 467,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 1000000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}