{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 400,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio": 0.0,
      "completion_length": 1000.0000610351562,
      "epoch": 0.0025,
      "grad_norm": 0.2079561913831216,
      "kl": 0.0158782958984375,
      "learning_rate": 3.3333333333333334e-08,
      "loss": 0.0,
      "num_tokens": 52920.0,
      "reward": 2.0,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.0,
      "rewards/slop_reward": 1.0,
      "step": 1
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 179.22857971191405,
      "epoch": 0.005,
      "grad_norm": 12.249017739002824,
      "kl": 0.00041656494140625,
      "learning_rate": 6.666666666666667e-08,
      "loss": 0.0,
      "num_tokens": 77113.0,
      "reward": 1.8515485525131226,
      "reward_std": 0.6024735510349274,
      "rewards/classifier_reward": 0.37967347651720046,
      "rewards/length_reward": 0.7142857313156128,
      "rewards/slop_reward": 0.7575892865657806,
      "step": 2
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 133.4857208251953,
      "epoch": 0.0075,
      "grad_norm": 10089.085994752151,
      "kl": 0.001357269287109375,
      "learning_rate": 1e-07,
      "loss": 0.0,
      "num_tokens": 99305.0,
      "reward": 1.5958443641662599,
      "reward_std": 0.44231254458427427,
      "rewards/classifier_reward": 0.4083442732691765,
      "rewards/length_reward": 0.3428571462631226,
      "rewards/slop_reward": 0.8446428537368774,
      "step": 3
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 215.08572387695312,
      "epoch": 0.01,
      "grad_norm": 2054.9417817778553,
      "kl": 0.0014644622802734374,
      "learning_rate": 1.3333333333333334e-07,
      "loss": 0.0,
      "num_tokens": 124753.0,
      "reward": 1.2784200072288514,
      "reward_std": 0.6290957629680634,
      "rewards/classifier_reward": 0.1494020951911807,
      "rewards/length_reward": 0.4571428656578064,
      "rewards/slop_reward": 0.6718749940395355,
      "step": 4
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 173.74286346435548,
      "epoch": 0.0125,
      "grad_norm": 2266.5233619504957,
      "kl": 0.001641082763671875,
      "learning_rate": 1.6666666666666665e-07,
      "loss": 0.0,
      "num_tokens": 148708.0,
      "reward": 1.3791944861412049,
      "reward_std": 0.43466432094573976,
      "rewards/classifier_reward": 0.17383726984262465,
      "rewards/length_reward": 0.5428571462631225,
      "rewards/slop_reward": 0.6625,
      "step": 5
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 202.02857971191406,
      "epoch": 0.015,
      "grad_norm": 11.643315762932316,
      "kl": 0.0005245208740234375,
      "learning_rate": 2e-07,
      "loss": 0.0,
      "num_tokens": 173615.0,
      "reward": 2.2288811445236205,
      "reward_std": 0.4513823240995407,
      "rewards/classifier_reward": 0.4342381663620472,
      "rewards/length_reward": 0.8857142925262451,
      "rewards/slop_reward": 0.9089285731315613,
      "step": 6
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 213.74286499023438,
      "epoch": 0.0175,
      "grad_norm": 13.650814377669256,
      "kl": 0.000527191162109375,
      "learning_rate": 2.3333333333333333e-07,
      "loss": 0.0,
      "num_tokens": 199016.0,
      "reward": 1.9626029968261718,
      "reward_std": 0.3459654450416565,
      "rewards/classifier_reward": 0.39831727296113967,
      "rewards/length_reward": 0.7142857164144516,
      "rewards/slop_reward": 0.8499999940395355,
      "step": 7
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 288.71429443359375,
      "epoch": 0.02,
      "grad_norm": 10.936857808142296,
      "kl": 0.0004474639892578125,
      "learning_rate": 2.6666666666666667e-07,
      "loss": 0.0,
      "num_tokens": 227041.0,
      "reward": 1.8008938789367677,
      "reward_std": 0.3342688336968422,
      "rewards/classifier_reward": 0.4794651668518782,
      "rewards/length_reward": 0.4285714328289032,
      "rewards/slop_reward": 0.8928571343421936,
      "step": 8
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 135.14286193847656,
      "epoch": 0.0225,
      "grad_norm": 13.869838227975858,
      "kl": 0.000525665283203125,
      "learning_rate": 3e-07,
      "loss": 0.0005,
      "num_tokens": 249691.0,
      "reward": 1.9354748964309691,
      "reward_std": 0.3361890375614166,
      "rewards/classifier_reward": 0.5604747980833054,
      "rewards/length_reward": 0.5142857193946838,
      "rewards/slop_reward": 0.8607142925262451,
      "step": 9
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 208.91429748535157,
      "epoch": 0.025,
      "grad_norm": 12.652344276830467,
      "kl": 0.0005481719970703125,
      "learning_rate": 3.333333333333333e-07,
      "loss": 0.0,
      "num_tokens": 274843.0,
      "reward": 1.943293523788452,
      "reward_std": 0.6581979870796204,
      "rewards/classifier_reward": 0.5843648463487625,
      "rewards/length_reward": 0.6000000178813935,
      "rewards/slop_reward": 0.7589285731315613,
      "step": 10
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 205.71429443359375,
      "epoch": 0.0275,
      "grad_norm": 15.071895621435214,
      "kl": 0.00064544677734375,
      "learning_rate": 3.666666666666666e-07,
      "loss": 0.0,
      "num_tokens": 299820.0,
      "reward": 1.5763698101043702,
      "reward_std": 0.4757842034101486,
      "rewards/classifier_reward": 0.431726861000061,
      "rewards/length_reward": 0.25714286863803865,
      "rewards/slop_reward": 0.8875,
      "step": 11
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 230.4285858154297,
      "epoch": 0.03,
      "grad_norm": 190.50276677565853,
      "kl": 0.014757537841796875,
      "learning_rate": 4e-07,
      "loss": 0.0,
      "num_tokens": 325730.0,
      "reward": 2.0366717338562013,
      "reward_std": 0.44329026341438293,
      "rewards/classifier_reward": 0.3331002712249756,
      "rewards/length_reward": 0.8857142925262451,
      "rewards/slop_reward": 0.8178571462631226,
      "step": 12
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 205.62857971191406,
      "epoch": 0.0325,
      "grad_norm": 12.86535673011762,
      "kl": 0.0007328033447265625,
      "learning_rate": 4.3333333333333335e-07,
      "loss": 0.0,
      "num_tokens": 350634.0,
      "reward": 1.3644737243652343,
      "reward_std": 0.5067215681076049,
      "rewards/classifier_reward": 0.29661653861403464,
      "rewards/length_reward": 0.31428571939468386,
      "rewards/slop_reward": 0.7535714268684387,
      "step": 13
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 149.25715026855468,
      "epoch": 0.035,
      "grad_norm": 13.332730561008187,
      "kl": 0.00087738037109375,
      "learning_rate": 4.6666666666666666e-07,
      "loss": 0.0,
      "num_tokens": 373261.0,
      "reward": 1.4552783489227294,
      "reward_std": 0.6117016971111298,
      "rewards/classifier_reward": 0.18384971991181373,
      "rewards/length_reward": 0.40000001192092893,
      "rewards/slop_reward": 0.8714285612106323,
      "step": 14
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 213.71429901123048,
      "epoch": 0.0375,
      "grad_norm": 12.15928533039066,
      "kl": 0.001068878173828125,
      "learning_rate": 5e-07,
      "loss": 0.0,
      "num_tokens": 398319.0,
      "reward": 1.7259351372718812,
      "reward_std": 0.5805591940879822,
      "rewards/classifier_reward": 0.5187922030687332,
      "rewards/length_reward": 0.48571428954601287,
      "rewards/slop_reward": 0.7214285731315613,
      "step": 15
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 200.94286499023437,
      "epoch": 0.04,
      "grad_norm": 210.10906643069993,
      "kl": 0.006036376953125,
      "learning_rate": 5.333333333333333e-07,
      "loss": 0.0,
      "num_tokens": 423267.0,
      "reward": 1.5157490253448487,
      "reward_std": 0.4267912685871124,
      "rewards/classifier_reward": 0.32646322101354597,
      "rewards/length_reward": 0.37142857611179353,
      "rewards/slop_reward": 0.8178571403026581,
      "step": 16
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 226.17143859863282,
      "epoch": 0.0425,
      "grad_norm": 10.900066010271587,
      "kl": 0.0018096923828125,
      "learning_rate": 5.666666666666666e-07,
      "loss": 0.0,
      "num_tokens": 449061.0,
      "reward": 1.5792103052139281,
      "reward_std": 0.5069970846176147,
      "rewards/classifier_reward": 0.13992448002099991,
      "rewards/length_reward": 0.6857142925262452,
      "rewards/slop_reward": 0.7535714209079742,
      "step": 17
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 167.08572387695312,
      "epoch": 0.045,
      "grad_norm": 265.89705750792126,
      "kl": 0.0086395263671875,
      "learning_rate": 6e-07,
      "loss": 0.0,
      "num_tokens": 472825.0,
      "reward": 1.8174922943115235,
      "reward_std": 0.39020195603370667,
      "rewards/classifier_reward": 0.3174922451376915,
      "rewards/length_reward": 0.6285714387893677,
      "rewards/slop_reward": 0.8714285731315613,
      "step": 18
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 150.85714721679688,
      "epoch": 0.0475,
      "grad_norm": 16.75575953327707,
      "kl": 0.0071929931640625,
      "learning_rate": 6.333333333333332e-07,
      "loss": 0.0,
      "num_tokens": 496025.0,
      "reward": 1.9033495664596558,
      "reward_std": 0.46038708090782166,
      "rewards/classifier_reward": 0.5676352053880691,
      "rewards/length_reward": 0.4000000059604645,
      "rewards/slop_reward": 0.9357142806053161,
      "step": 19
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 170.20000610351562,
      "epoch": 0.05,
      "grad_norm": 12.507998039364391,
      "kl": 0.008447265625,
      "learning_rate": 6.666666666666666e-07,
      "loss": 0.0,
      "num_tokens": 519775.0,
      "reward": 1.8573363780975343,
      "reward_std": 0.46103876233100893,
      "rewards/classifier_reward": 0.3716219961643219,
      "rewards/length_reward": 0.6571428596973419,
      "rewards/slop_reward": 0.8285714149475097,
      "step": 20
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 254.57144317626953,
      "epoch": 0.0525,
      "grad_norm": 10.08679593336001,
      "kl": 0.005413818359375,
      "learning_rate": 7e-07,
      "loss": 0.0,
      "num_tokens": 546566.0,
      "reward": 1.2423678815364838,
      "reward_std": 0.4237713754177094,
      "rewards/classifier_reward": 0.2472785457968712,
      "rewards/length_reward": 0.45714286267757415,
      "rewards/slop_reward": 0.5379464238882065,
      "step": 21
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 231.60000915527343,
      "epoch": 0.055,
      "grad_norm": 120.69825805503362,
      "kl": 0.01463623046875,
      "learning_rate": 7.333333333333332e-07,
      "loss": 0.0,
      "num_tokens": 572155.0,
      "reward": 1.6102877855300903,
      "reward_std": 0.47996904850006106,
      "rewards/classifier_reward": 0.39421619176864625,
      "rewards/length_reward": 0.45714286267757415,
      "rewards/slop_reward": 0.7589285731315613,
      "step": 22
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 244.971435546875,
      "epoch": 0.0575,
      "grad_norm": 10.506951478641199,
      "kl": 0.0114501953125,
      "learning_rate": 7.666666666666667e-07,
      "loss": 0.0,
      "num_tokens": 598457.0,
      "reward": 1.7265684366226197,
      "reward_std": 0.5010857343673706,
      "rewards/classifier_reward": 0.35513979494571685,
      "rewards/length_reward": 0.542857152223587,
      "rewards/slop_reward": 0.8285714268684388,
      "step": 23
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 212.17143859863282,
      "epoch": 0.06,
      "grad_norm": 14.496994349875411,
      "kl": 0.019873046875,
      "learning_rate": 8e-07,
      "loss": 0.0,
      "num_tokens": 623795.0,
      "reward": 1.9756683349609374,
      "reward_std": 0.6098839461803436,
      "rewards/classifier_reward": 0.3970968216657639,
      "rewards/length_reward": 0.6857142925262452,
      "rewards/slop_reward": 0.8928571343421936,
      "step": 24
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 149.71429138183595,
      "epoch": 0.0625,
      "grad_norm": 13.955034723905843,
      "kl": 0.023046875,
      "learning_rate": 8.333333333333333e-07,
      "loss": 0.0,
      "num_tokens": 646708.0,
      "reward": 1.7968461036682128,
      "reward_std": 0.45076006054878237,
      "rewards/classifier_reward": 0.3789888650178909,
      "rewards/length_reward": 0.5142857193946838,
      "rewards/slop_reward": 0.9035714268684387,
      "step": 25
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 172.02857818603516,
      "epoch": 0.065,
      "grad_norm": 13.086801218372846,
      "kl": 0.03857421875,
      "learning_rate": 8.666666666666667e-07,
      "loss": 0.0,
      "num_tokens": 670238.0,
      "reward": 2.018597435951233,
      "reward_std": 0.5030077040195465,
      "rewards/classifier_reward": 0.44716874957084657,
      "rewards/length_reward": 0.6571428656578064,
      "rewards/slop_reward": 0.9142857074737549,
      "step": 26
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 139.0571487426758,
      "epoch": 0.0675,
      "grad_norm": 13.874334958965811,
      "kl": 0.044140625,
      "learning_rate": 9e-07,
      "loss": 0.0,
      "num_tokens": 693025.0,
      "reward": 1.3813447833061219,
      "reward_std": 0.4743997871875763,
      "rewards/classifier_reward": 0.3313447292894125,
      "rewards/length_reward": 0.28571428954601286,
      "rewards/slop_reward": 0.7642857193946838,
      "step": 27
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 312.0857269287109,
      "epoch": 0.07,
      "grad_norm": 9.049982563385965,
      "kl": 0.0622314453125,
      "learning_rate": 9.333333333333333e-07,
      "loss": 0.0001,
      "num_tokens": 721725.0,
      "reward": 1.688427746295929,
      "reward_std": 0.4802214980125427,
      "rewards/classifier_reward": 0.48485626801848414,
      "rewards/length_reward": 0.5142857193946838,
      "rewards/slop_reward": 0.6892857074737548,
      "step": 28
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 170.34286499023438,
      "epoch": 0.0725,
      "grad_norm": 13.386821366059916,
      "kl": 0.100341796875,
      "learning_rate": 9.666666666666666e-07,
      "loss": 0.0001,
      "num_tokens": 745420.0,
      "reward": 1.488853096961975,
      "reward_std": 0.4731449127197266,
      "rewards/classifier_reward": 0.2870673179626465,
      "rewards/length_reward": 0.31428571939468386,
      "rewards/slop_reward": 0.8875,
      "step": 29
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 250.0571502685547,
      "epoch": 0.075,
      "grad_norm": 11.769241786475325,
      "kl": 0.08681640625,
      "learning_rate": 1e-06,
      "loss": 0.0001,
      "num_tokens": 771967.0,
      "reward": 1.6088370084762573,
      "reward_std": 0.5624471366405487,
      "rewards/classifier_reward": 0.2784797720611095,
      "rewards/length_reward": 0.4857142925262451,
      "rewards/slop_reward": 0.8446428537368774,
      "step": 30
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 203.6285858154297,
      "epoch": 0.0775,
      "grad_norm": 15.786130440761754,
      "kl": 0.120703125,
      "learning_rate": 9.999819767255174e-07,
      "loss": 0.0001,
      "num_tokens": 797014.0,
      "reward": 1.9446904182434082,
      "reward_std": 0.36648078858852384,
      "rewards/classifier_reward": 0.4661188304424286,
      "rewards/length_reward": 0.5428571462631225,
      "rewards/slop_reward": 0.9357142806053161,
      "step": 31
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 201.771435546875,
      "epoch": 0.08,
      "grad_norm": 15.903852139666391,
      "kl": 0.3494140625,
      "learning_rate": 9.999279082014231e-07,
      "loss": 0.0003,
      "num_tokens": 821995.0,
      "reward": 1.9787015676498414,
      "reward_std": 0.585156524181366,
      "rewards/classifier_reward": 0.5215587019920349,
      "rewards/length_reward": 0.5428571552038193,
      "rewards/slop_reward": 0.9142857074737549,
      "step": 32
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 224.42858276367187,
      "epoch": 0.0825,
      "grad_norm": 29.64188533235313,
      "kl": 0.500390625,
      "learning_rate": 9.998377983256848e-07,
      "loss": 0.0005,
      "num_tokens": 847628.0,
      "reward": 2.037756896018982,
      "reward_std": 0.5854993224143982,
      "rewards/classifier_reward": 0.45918539762496946,
      "rewards/length_reward": 0.6857142984867096,
      "rewards/slop_reward": 0.8928571343421936,
      "step": 33
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 213.91429443359374,
      "epoch": 0.085,
      "grad_norm": 13.883875104448363,
      "kl": 1.748828125,
      "learning_rate": 9.997116535946027e-07,
      "loss": 0.0018,
      "num_tokens": 872871.0,
      "reward": 1.7733258247375487,
      "reward_std": 0.5001925647258758,
      "rewards/classifier_reward": 0.4197543442249298,
      "rewards/length_reward": 0.6857142955064773,
      "rewards/slop_reward": 0.6678571343421936,
      "step": 34
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 229.771435546875,
      "epoch": 0.0875,
      "grad_norm": 14.166455798732835,
      "kl": 1.48515625,
      "learning_rate": 9.995494831023408e-07,
      "loss": 0.0015,
      "num_tokens": 898833.0,
      "reward": 2.2283714771270753,
      "reward_std": 0.4383322179317474,
      "rewards/classifier_reward": 0.6283714175224304,
      "rewards/length_reward": 0.6857142865657806,
      "rewards/slop_reward": 0.9142857074737549,
      "step": 35
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 240.5428680419922,
      "epoch": 0.09,
      "grad_norm": 20.07276091542321,
      "kl": 1.05703125,
      "learning_rate": 9.993512985402724e-07,
      "loss": 0.0011,
      "num_tokens": 924948.0,
      "reward": 1.8037936687469482,
      "reward_std": 0.4002750262618065,
      "rewards/classifier_reward": 0.4395078897476196,
      "rewards/length_reward": 0.4285714328289032,
      "rewards/slop_reward": 0.9357142806053161,
      "step": 36
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 249.42858123779297,
      "epoch": 0.0925,
      "grad_norm": 16.81239416666408,
      "kl": 7.96875,
      "learning_rate": 9.991171141961368e-07,
      "loss": 0.0085,
      "num_tokens": 951092.0,
      "reward": 1.9397377490997314,
      "reward_std": 0.4508820950984955,
      "rewards/classifier_reward": 0.6325947523117066,
      "rewards/length_reward": 0.4571428656578064,
      "rewards/slop_reward": 0.8499999880790711,
      "step": 37
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 196.17143707275392,
      "epoch": 0.095,
      "grad_norm": 19.19953442548882,
      "kl": 3.775,
      "learning_rate": 9.988469469530085e-07,
      "loss": 0.0038,
      "num_tokens": 975652.0,
      "reward": 2.0388022661209106,
      "reward_std": 0.3248827219009399,
      "rewards/classifier_reward": 0.32451642835512756,
      "rewards/length_reward": 0.7142857193946839,
      "rewards/slop_reward": 1.0,
      "step": 38
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 352.22858276367185,
      "epoch": 0.0975,
      "grad_norm": 18.25441612913441,
      "kl": 0.79091796875,
      "learning_rate": 9.985408162880813e-07,
      "loss": 0.0008,
      "num_tokens": 1005900.0,
      "reward": 1.6981183767318726,
      "reward_std": 0.433578160405159,
      "rewards/classifier_reward": 0.5351718068122864,
      "rewards/length_reward": 0.40000000298023225,
      "rewards/slop_reward": 0.7629464268684387,
      "step": 39
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 321.02858276367186,
      "epoch": 0.1,
      "grad_norm": 10.54620387526757,
      "kl": 0.754296875,
      "learning_rate": 9.98198744271263e-07,
      "loss": 0.0008,
      "num_tokens": 1034877.0,
      "reward": 1.9676434993743896,
      "reward_std": 0.4024129122495651,
      "rewards/classifier_reward": 0.6390719175338745,
      "rewards/length_reward": 0.4571428656578064,
      "rewards/slop_reward": 0.8714285612106323,
      "step": 40
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 214.17143859863282,
      "epoch": 0.1025,
      "grad_norm": 13.824754472854506,
      "kl": 0.911328125,
      "learning_rate": 9.978207555635855e-07,
      "loss": 0.0009,
      "num_tokens": 1060157.0,
      "reward": 2.1357697248458862,
      "reward_std": 0.5395443201065063,
      "rewards/classifier_reward": 0.6357696294784546,
      "rewards/length_reward": 0.5428571552038193,
      "rewards/slop_reward": 0.9571428537368775,
      "step": 41
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 223.6285827636719,
      "epoch": 0.105,
      "grad_norm": 18.073667394788664,
      "kl": 0.519921875,
      "learning_rate": 9.97406877415425e-07,
      "loss": 0.0005,
      "num_tokens": 1085893.0,
      "reward": 2.068192982673645,
      "reward_std": 0.4686335951089859,
      "rewards/classifier_reward": 0.4967642992734909,
      "rewards/length_reward": 0.6571428775787354,
      "rewards/slop_reward": 0.9142857074737549,
      "step": 42
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 297.40001220703124,
      "epoch": 0.1075,
      "grad_norm": 10.8725727731831,
      "kl": 0.43515625,
      "learning_rate": 9.9695713966454e-07,
      "loss": 0.0004,
      "num_tokens": 1114056.0,
      "reward": 1.6727801322937013,
      "reward_std": 0.501282411813736,
      "rewards/classifier_reward": 0.501351535320282,
      "rewards/length_reward": 0.3428571492433548,
      "rewards/slop_reward": 0.8285714149475097,
      "step": 43
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 204.85715026855468,
      "epoch": 0.11,
      "grad_norm": 11.442387542173964,
      "kl": 0.574609375,
      "learning_rate": 9.964715747339175e-07,
      "loss": 0.0006,
      "num_tokens": 1138804.0,
      "reward": 2.027732276916504,
      "reward_std": 0.6377828001976014,
      "rewards/classifier_reward": 0.6545179545879364,
      "rewards/length_reward": 0.48571430146694183,
      "rewards/slop_reward": 0.8875,
      "step": 44
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 290.6285888671875,
      "epoch": 0.1125,
      "grad_norm": 11.010885455058528,
      "kl": 0.451171875,
      "learning_rate": 9.959502176294382e-07,
      "loss": 0.0005,
      "num_tokens": 1166842.0,
      "reward": 1.8717997074127197,
      "reward_std": 0.45424606651067734,
      "rewards/classifier_reward": 0.45037112236022947,
      "rewards/length_reward": 0.4857142925262451,
      "rewards/slop_reward": 0.9357142806053161,
      "step": 45
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 251.60001220703126,
      "epoch": 0.115,
      "grad_norm": 10.74794734294439,
      "kl": 0.378125,
      "learning_rate": 9.953931059373511e-07,
      "loss": 0.0004,
      "num_tokens": 1193568.0,
      "reward": 2.1009025812149047,
      "reward_std": 0.5576321303844451,
      "rewards/classifier_reward": 0.6455453038215637,
      "rewards/length_reward": 0.6000000059604644,
      "rewards/slop_reward": 0.8553571462631225,
      "step": 46
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 205.51429595947266,
      "epoch": 0.1175,
      "grad_norm": 12.174860136090478,
      "kl": 0.580859375,
      "learning_rate": 9.948002798215631e-07,
      "loss": 0.0006,
      "num_tokens": 1218520.0,
      "reward": 1.7478339910507201,
      "reward_std": 0.44800390899181364,
      "rewards/classifier_reward": 0.5246196419000626,
      "rewards/length_reward": 0.4000000059604645,
      "rewards/slop_reward": 0.8232142806053162,
      "step": 47
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 206.371435546875,
      "epoch": 0.12,
      "grad_norm": 10.734345225393307,
      "kl": 0.77734375,
      "learning_rate": 9.94171782020746e-07,
      "loss": 0.0008,
      "num_tokens": 1243553.0,
      "reward": 2.33663330078125,
      "reward_std": 0.5126991689205169,
      "rewards/classifier_reward": 0.6152046620845795,
      "rewards/length_reward": 0.7428571522235871,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 48
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 349.8000244140625,
      "epoch": 0.1225,
      "grad_norm": 9.516105487183257,
      "kl": 0.6921875,
      "learning_rate": 9.935076578452534e-07,
      "loss": 0.0007,
      "num_tokens": 1273677.0,
      "reward": 1.6252358913421632,
      "reward_std": 0.4947394013404846,
      "rewards/classifier_reward": 0.5180929381400347,
      "rewards/length_reward": 0.25714286267757414,
      "rewards/slop_reward": 0.85,
      "step": 49
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 229.57143859863282,
      "epoch": 0.125,
      "grad_norm": 11.582425399894849,
      "kl": 1.07734375,
      "learning_rate": 9.928079551738541e-07,
      "loss": 0.0011,
      "num_tokens": 1299334.0,
      "reward": 2.2410747528076174,
      "reward_std": 0.5065959393978119,
      "rewards/classifier_reward": 0.5834853827953339,
      "rewards/length_reward": 0.6857143044471741,
      "rewards/slop_reward": 0.971875,
      "step": 50
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 254.68572692871095,
      "epoch": 0.1275,
      "grad_norm": 9.691651370929392,
      "kl": 0.7765625,
      "learning_rate": 9.920727244502818e-07,
      "loss": 0.0008,
      "num_tokens": 1326112.0,
      "reward": 1.8951802968978881,
      "reward_std": 0.561316728591919,
      "rewards/classifier_reward": 0.4666088119149208,
      "rewards/length_reward": 0.5142857283353806,
      "rewards/slop_reward": 0.9142857074737549,
      "step": 51
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 245.82858581542968,
      "epoch": 0.13,
      "grad_norm": 9.231142781375425,
      "kl": 0.4640625,
      "learning_rate": 9.913020186795966e-07,
      "loss": 0.0005,
      "num_tokens": 1352635.0,
      "reward": 2.4359071254730225,
      "reward_std": 0.504255086183548,
      "rewards/classifier_reward": 0.8644783020019531,
      "rewards/length_reward": 0.6571428656578064,
      "rewards/slop_reward": 0.9142857074737549,
      "step": 52
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 336.2857360839844,
      "epoch": 0.1325,
      "grad_norm": 8.070502355443343,
      "kl": 1.01142578125,
      "learning_rate": 9.904958934243653e-07,
      "loss": 0.001,
      "num_tokens": 1382325.0,
      "reward": 1.9690003395080566,
      "reward_std": 0.4140917003154755,
      "rewards/classifier_reward": 0.595339572429657,
      "rewards/length_reward": 0.5142857253551483,
      "rewards/slop_reward": 0.859375,
      "step": 53
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 176.85714721679688,
      "epoch": 0.135,
      "grad_norm": 13.273584784804553,
      "kl": 0.737109375,
      "learning_rate": 9.89654406800655e-07,
      "loss": 0.0007,
      "num_tokens": 1406364.0,
      "reward": 2.015079474449158,
      "reward_std": 0.34893424808979034,
      "rewards/classifier_reward": 0.6650793373584747,
      "rewards/length_reward": 0.37142857611179353,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 54
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 270.77144317626954,
      "epoch": 0.1375,
      "grad_norm": 10.259811273773323,
      "kl": 0.41875,
      "learning_rate": 9.887776194738431e-07,
      "loss": 0.0004,
      "num_tokens": 1433517.0,
      "reward": 2.0523552179336546,
      "reward_std": 0.43476098477840425,
      "rewards/classifier_reward": 0.72378368973732,
      "rewards/length_reward": 0.37142857909202576,
      "rewards/slop_reward": 0.9571428537368775,
      "step": 55
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 270.20001220703125,
      "epoch": 0.14,
      "grad_norm": 98.33006992571028,
      "kl": 25.924609375,
      "learning_rate": 9.878655946542442e-07,
      "loss": 0.0258,
      "num_tokens": 1460894.0,
      "reward": 2.2650604486465453,
      "reward_std": 0.5314578056335449,
      "rewards/classifier_reward": 0.7382745862007141,
      "rewards/length_reward": 0.6285714358091354,
      "rewards/slop_reward": 0.8982142806053162,
      "step": 56
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 213.82858581542968,
      "epoch": 0.1425,
      "grad_norm": 11.336151350522224,
      "kl": 0.592578125,
      "learning_rate": 9.86918398092553e-07,
      "loss": 0.0006,
      "num_tokens": 1486239.0,
      "reward": 2.248945116996765,
      "reward_std": 0.4141096830368042,
      "rewards/classifier_reward": 0.6918022215366364,
      "rewards/length_reward": 0.6000000059604644,
      "rewards/slop_reward": 0.9571428537368775,
      "step": 57
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 301.85715637207034,
      "epoch": 0.145,
      "grad_norm": 9.956188328948622,
      "kl": 0.44765625,
      "learning_rate": 9.85936098075104e-07,
      "loss": 0.0004,
      "num_tokens": 1514661.0,
      "reward": 1.9887679100036622,
      "reward_std": 0.47130251824855807,
      "rewards/classifier_reward": 0.6173391878604889,
      "rewards/length_reward": 0.45714286267757415,
      "rewards/slop_reward": 0.9142857074737549,
      "step": 58
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 194.42857971191407,
      "epoch": 0.1475,
      "grad_norm": 10.413710834913637,
      "kl": 0.428125,
      "learning_rate": 9.849187654189485e-07,
      "loss": 0.0004,
      "num_tokens": 1539249.0,
      "reward": 2.1201124668121336,
      "reward_std": 0.4662812829017639,
      "rewards/classifier_reward": 0.8486838459968566,
      "rewards/length_reward": 0.3142857253551483,
      "rewards/slop_reward": 0.9571428537368775,
      "step": 59
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 223.71429443359375,
      "epoch": 0.15,
      "grad_norm": 9.48253169035506,
      "kl": 0.6697265625,
      "learning_rate": 9.838664734667495e-07,
      "loss": 0.0007,
      "num_tokens": 1564932.0,
      "reward": 2.3097215414047243,
      "reward_std": 0.37846060991287234,
      "rewards/classifier_reward": 0.6097213685512543,
      "rewards/length_reward": 0.7428571492433548,
      "rewards/slop_reward": 0.9571428537368775,
      "step": 60
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 213.08572235107422,
      "epoch": 0.1525,
      "grad_norm": 17.056291850314448,
      "kl": 0.484375,
      "learning_rate": 9.827792980814933e-07,
      "loss": 0.0005,
      "num_tokens": 1590245.0,
      "reward": 2.139114594459534,
      "reward_std": 0.3371311604976654,
      "rewards/classifier_reward": 0.6605431139469147,
      "rewards/length_reward": 0.5428571492433548,
      "rewards/slop_reward": 0.9357142806053161,
      "step": 61
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 224.68572540283202,
      "epoch": 0.155,
      "grad_norm": 9.724576982946711,
      "kl": 0.580859375,
      "learning_rate": 9.81657317641022e-07,
      "loss": 0.0006,
      "num_tokens": 1615704.0,
      "reward": 2.022816562652588,
      "reward_std": 0.4144854575395584,
      "rewards/classifier_reward": 0.5585307866334915,
      "rewards/length_reward": 0.4857142955064774,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 62
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 343.8285888671875,
      "epoch": 0.1575,
      "grad_norm": 7.143874630694377,
      "kl": 0.4015625,
      "learning_rate": 9.805006130323808e-07,
      "loss": 0.0004,
      "num_tokens": 1645654.0,
      "reward": 2.0719400882720946,
      "reward_std": 0.3876799166202545,
      "rewards/classifier_reward": 0.6647971898317337,
      "rewards/length_reward": 0.42857143878936765,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 63
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 266.00001220703126,
      "epoch": 0.16,
      "grad_norm": 9.518081765887688,
      "kl": 0.3986328125,
      "learning_rate": 9.793092676459888e-07,
      "loss": 0.0004,
      "num_tokens": 1672883.0,
      "reward": 2.084528160095215,
      "reward_std": 0.473650124669075,
      "rewards/classifier_reward": 0.584528061747551,
      "rewards/length_reward": 0.5428571552038193,
      "rewards/slop_reward": 0.9571428537368775,
      "step": 64
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 279.71429748535155,
      "epoch": 0.1625,
      "grad_norm": 8.015308302511995,
      "kl": 0.54375,
      "learning_rate": 9.780833673696254e-07,
      "loss": 0.0005,
      "num_tokens": 1700438.0,
      "reward": 2.3380573272705076,
      "reward_std": 0.4121220216155052,
      "rewards/classifier_reward": 0.8237714767456055,
      "rewards/length_reward": 0.5142857223749161,
      "rewards/slop_reward": 1.0,
      "step": 65
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 248.31430053710938,
      "epoch": 0.165,
      "grad_norm": 8.749621821849056,
      "kl": 0.421484375,
      "learning_rate": 9.768230005822393e-07,
      "loss": 0.0004,
      "num_tokens": 1727047.0,
      "reward": 2.2453027963638306,
      "reward_std": 0.38129588067531583,
      "rewards/classifier_reward": 0.781016880273819,
      "rewards/length_reward": 0.48571428954601287,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 66
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 283.3142974853516,
      "epoch": 0.1675,
      "grad_norm": 15.835663084851635,
      "kl": 0.4734375,
      "learning_rate": 9.755282581475767e-07,
      "loss": 0.0005,
      "num_tokens": 1754863.0,
      "reward": 2.350207304954529,
      "reward_std": 0.5121555209159852,
      "rewards/classifier_reward": 0.7930643558502197,
      "rewards/length_reward": 0.6000000149011612,
      "rewards/slop_reward": 0.9571428537368775,
      "step": 67
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 207.0571502685547,
      "epoch": 0.17,
      "grad_norm": 10.560189239731793,
      "kl": 0.418359375,
      "learning_rate": 9.741992334076308e-07,
      "loss": 0.0004,
      "num_tokens": 1780017.0,
      "reward": 2.56486029624939,
      "reward_std": 0.4183764517307281,
      "rewards/classifier_reward": 0.7505744874477387,
      "rewards/length_reward": 0.8571428656578064,
      "rewards/slop_reward": 0.9571428537368775,
      "step": 68
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 207.31429443359374,
      "epoch": 0.1725,
      "grad_norm": 12.483812322806989,
      "kl": 0.690234375,
      "learning_rate": 9.728360221759123e-07,
      "loss": 0.0007,
      "num_tokens": 1805172.0,
      "reward": 2.421455478668213,
      "reward_std": 0.5775963604450226,
      "rewards/classifier_reward": 0.7643125176429748,
      "rewards/length_reward": 0.7428571522235871,
      "rewards/slop_reward": 0.9142857074737549,
      "step": 69
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 262.8285858154297,
      "epoch": 0.175,
      "grad_norm": 10.184314056992156,
      "kl": 0.665234375,
      "learning_rate": 9.71438722730542e-07,
      "loss": 0.0007,
      "num_tokens": 1832291.0,
      "reward": 2.078964352607727,
      "reward_std": 0.5653822362422943,
      "rewards/classifier_reward": 0.6146785497665406,
      "rewards/length_reward": 0.4857142984867096,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 70
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 338.9428741455078,
      "epoch": 0.1775,
      "grad_norm": 8.806960986372623,
      "kl": 0.46015625,
      "learning_rate": 9.700074358071656e-07,
      "loss": 0.0005,
      "num_tokens": 1861997.0,
      "reward": 2.0550333499908446,
      "reward_std": 0.5070074677467347,
      "rewards/classifier_reward": 0.7263502657413483,
      "rewards/length_reward": 0.40000001192092893,
      "rewards/slop_reward": 0.9286830306053162,
      "step": 71
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 270.0857299804687,
      "epoch": 0.18,
      "grad_norm": 9.059350132300025,
      "kl": 0.450390625,
      "learning_rate": 9.685422645916918e-07,
      "loss": 0.0005,
      "num_tokens": 1888854.0,
      "reward": 2.531459331512451,
      "reward_std": 0.3892214775085449,
      "rewards/classifier_reward": 0.860030734539032,
      "rewards/length_reward": 0.7142857193946839,
      "rewards/slop_reward": 0.9571428537368775,
      "step": 72
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 291.4285827636719,
      "epoch": 0.1825,
      "grad_norm": 12.813964228104469,
      "kl": 0.52265625,
      "learning_rate": 9.670433147128521e-07,
      "loss": 0.0005,
      "num_tokens": 1916974.0,
      "reward": 2.2648436784744264,
      "reward_std": 0.40854659080505373,
      "rewards/classifier_reward": 0.8577007412910461,
      "rewards/length_reward": 0.42857143878936765,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 73
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 246.31429748535157,
      "epoch": 0.185,
      "grad_norm": 241.73474838722862,
      "kl": 0.453125,
      "learning_rate": 9.655106942345868e-07,
      "loss": 0.0005,
      "num_tokens": 1943440.0,
      "reward": 2.366797590255737,
      "reward_std": 0.41730722188949587,
      "rewards/classifier_reward": 0.766797399520874,
      "rewards/length_reward": 0.600000011920929,
      "rewards/slop_reward": 1.0,
      "step": 74
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 250.91429443359374,
      "epoch": 0.1875,
      "grad_norm": 8.395335843546244,
      "kl": 0.5015625,
      "learning_rate": 9.639445136482546e-07,
      "loss": 0.0005,
      "num_tokens": 1969900.0,
      "reward": 2.4618414878845214,
      "reward_std": 0.540682977437973,
      "rewards/classifier_reward": 0.8261271595954895,
      "rewards/length_reward": 0.6571428716182709,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 75
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 247.11429748535156,
      "epoch": 0.19,
      "grad_norm": 9.31969072416839,
      "kl": 0.53671875,
      "learning_rate": 9.623448858646656e-07,
      "loss": 0.0005,
      "num_tokens": 1996327.0,
      "reward": 2.259377145767212,
      "reward_std": 0.4671668648719788,
      "rewards/classifier_reward": 0.916519820690155,
      "rewards/length_reward": 0.3428571581840515,
      "rewards/slop_reward": 1.0,
      "step": 76
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 316.9714416503906,
      "epoch": 0.1925,
      "grad_norm": 7.365001527017238,
      "kl": 1.408984375,
      "learning_rate": 9.607119262059425e-07,
      "loss": 0.0014,
      "num_tokens": 2024968.0,
      "reward": 2.266000509262085,
      "reward_std": 0.5217303335666656,
      "rewards/classifier_reward": 0.8231433391571045,
      "rewards/length_reward": 0.48571430146694183,
      "rewards/slop_reward": 0.9571428537368775,
      "step": 77
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 228.62857971191406,
      "epoch": 0.195,
      "grad_norm": 45.20061919834423,
      "kl": 23.44453125,
      "learning_rate": 9.590457523972055e-07,
      "loss": 0.0236,
      "num_tokens": 2050735.0,
      "reward": 2.3131242275238035,
      "reward_std": 0.5057340741157532,
      "rewards/classifier_reward": 0.8827670216560364,
      "rewards/length_reward": 0.4571428656578064,
      "rewards/slop_reward": 0.9732142806053161,
      "step": 78
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 217.91429443359374,
      "epoch": 0.1975,
      "grad_norm": 9.435990839066255,
      "kl": 0.4484375,
      "learning_rate": 9.573464845580863e-07,
      "loss": 0.0004,
      "num_tokens": 2076160.0,
      "reward": 2.3861000537872314,
      "reward_std": 0.6137513637542724,
      "rewards/classifier_reward": 0.8432427525520325,
      "rewards/length_reward": 0.5428571581840516,
      "rewards/slop_reward": 1.0,
      "step": 79
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 250.91429443359374,
      "epoch": 0.2,
      "grad_norm": 7.98212579749696,
      "kl": 0.5078125,
      "learning_rate": 9.556142451940679e-07,
      "loss": 0.0005,
      "num_tokens": 2102862.0,
      "reward": 2.2402546644210815,
      "reward_std": 0.602302199602127,
      "rewards/classifier_reward": 0.7473974108695984,
      "rewards/length_reward": 0.600000011920929,
      "rewards/slop_reward": 0.8928571343421936,
      "step": 80
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 287.34287414550784,
      "epoch": 0.2025,
      "grad_norm": 8.04736588295688,
      "kl": 0.61015625,
      "learning_rate": 9.53849159187652e-07,
      "loss": 0.0006,
      "num_tokens": 2130556.0,
      "reward": 2.3916960954666138,
      "reward_std": 0.45145381689071656,
      "rewards/classifier_reward": 0.9631245970726013,
      "rewards/length_reward": 0.5142857253551483,
      "rewards/slop_reward": 0.9142857074737549,
      "step": 81
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 264.02857666015626,
      "epoch": 0.205,
      "grad_norm": 8.203747363662693,
      "kl": 0.523828125,
      "learning_rate": 9.520513537893573e-07,
      "loss": 0.0005,
      "num_tokens": 2157656.0,
      "reward": 2.4363236665725707,
      "reward_std": 0.32723597437143326,
      "rewards/classifier_reward": 0.7720378637313843,
      "rewards/length_reward": 0.6857142925262452,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 82
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 229.02858276367186,
      "epoch": 0.2075,
      "grad_norm": 7.89698513820585,
      "kl": 0.556640625,
      "learning_rate": 9.502209586085442e-07,
      "loss": 0.0006,
      "num_tokens": 2183592.0,
      "reward": 2.6267528533935547,
      "reward_std": 0.41746888160705564,
      "rewards/classifier_reward": 0.9338955879211426,
      "rewards/length_reward": 0.7142857253551483,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 83
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 224.37144165039064,
      "epoch": 0.21,
      "grad_norm": 8.428058041901203,
      "kl": 0.534765625,
      "learning_rate": 9.483581056040718e-07,
      "loss": 0.0005,
      "num_tokens": 2209201.0,
      "reward": 2.4542306900024413,
      "reward_std": 0.4101273000240326,
      "rewards/classifier_reward": 0.9113734126091003,
      "rewards/length_reward": 0.5428571552038193,
      "rewards/slop_reward": 1.0,
      "step": 84
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 257.71429443359375,
      "epoch": 0.2125,
      "grad_norm": 9.035933890702934,
      "kl": 0.5140625,
      "learning_rate": 9.464629290747842e-07,
      "loss": 0.0005,
      "num_tokens": 2236141.0,
      "reward": 2.5825096130371095,
      "reward_std": 0.45454559922218324,
      "rewards/classifier_reward": 0.9753666520118713,
      "rewards/length_reward": 0.6285714417695999,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 85
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 246.74286804199218,
      "epoch": 0.215,
      "grad_norm": 9.318935519161128,
      "kl": 0.555078125,
      "learning_rate": 9.445355656498284e-07,
      "loss": 0.0006,
      "num_tokens": 2262697.0,
      "reward": 2.383002519607544,
      "reward_std": 0.5029416978359222,
      "rewards/classifier_reward": 0.8830024480819703,
      "rewards/length_reward": 0.5428571611642837,
      "rewards/slop_reward": 0.9571428537368775,
      "step": 86
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 324.9428741455078,
      "epoch": 0.2175,
      "grad_norm": 10.476800992230684,
      "kl": 0.515234375,
      "learning_rate": 9.425761542788048e-07,
      "loss": 0.001,
      "num_tokens": 2291611.0,
      "reward": 2.4253102779388427,
      "reward_std": 0.25630177855491637,
      "rewards/classifier_reward": 0.9878101944923401,
      "rewards/length_reward": 0.4857142955064774,
      "rewards/slop_reward": 0.9517857193946838,
      "step": 87
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 285.88572387695314,
      "epoch": 0.22,
      "grad_norm": 8.262503873333891,
      "kl": 0.623828125,
      "learning_rate": 9.40584836221749e-07,
      "loss": 0.0006,
      "num_tokens": 2319400.0,
      "reward": 2.459568977355957,
      "reward_std": 0.5227903485298157,
      "rewards/classifier_reward": 0.9595688700675964,
      "rewards/length_reward": 0.5428571581840516,
      "rewards/slop_reward": 0.9571428537368775,
      "step": 88
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 361.48572998046876,
      "epoch": 0.2225,
      "grad_norm": 8.639609161679585,
      "kl": 0.61015625,
      "learning_rate": 9.385617550389489e-07,
      "loss": 0.0006,
      "num_tokens": 2349972.0,
      "reward": 2.21625759601593,
      "reward_std": 0.43572868704795836,
      "rewards/classifier_reward": 0.8430432677268982,
      "rewards/length_reward": 0.4000000089406967,
      "rewards/slop_reward": 0.9732142806053161,
      "step": 89
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 230.5428680419922,
      "epoch": 0.225,
      "grad_norm": 9.545010126368608,
      "kl": 0.606640625,
      "learning_rate": 9.36507056580594e-07,
      "loss": 0.0006,
      "num_tokens": 2375941.0,
      "reward": 2.5845241069793703,
      "reward_std": 0.4674242250621319,
      "rewards/classifier_reward": 0.9559526205062866,
      "rewards/length_reward": 0.6285714387893677,
      "rewards/slop_reward": 1.0,
      "step": 90
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 255.971435546875,
      "epoch": 0.2275,
      "grad_norm": 8.313767260844978,
      "kl": 0.705859375,
      "learning_rate": 9.34420888976262e-07,
      "loss": 0.0007,
      "num_tokens": 2402820.0,
      "reward": 2.281861972808838,
      "reward_std": 0.5015565395355225,
      "rewards/classifier_reward": 0.9032904744148255,
      "rewards/length_reward": 0.48571430444717406,
      "rewards/slop_reward": 0.8928571343421936,
      "step": 91
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 285.71430053710935,
      "epoch": 0.23,
      "grad_norm": 15.296685695089757,
      "kl": 12.5765625,
      "learning_rate": 9.323034026242377e-07,
      "loss": 0.0126,
      "num_tokens": 2430740.0,
      "reward": 2.457769823074341,
      "reward_std": 0.4592562437057495,
      "rewards/classifier_reward": 0.9363411664962769,
      "rewards/length_reward": 0.542857152223587,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 92
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 283.9143035888672,
      "epoch": 0.2325,
      "grad_norm": 41.32916438788783,
      "kl": 0.500390625,
      "learning_rate": 9.301547501806724e-07,
      "loss": 0.0005,
      "num_tokens": 2458445.0,
      "reward": 2.432806062698364,
      "reward_std": 0.4543603718280792,
      "rewards/classifier_reward": 0.8256630301475525,
      "rewards/length_reward": 0.6285714387893677,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 93
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 231.2571533203125,
      "epoch": 0.235,
      "grad_norm": 8.514937250494981,
      "kl": 0.56875,
      "learning_rate": 9.279750865485772e-07,
      "loss": 0.001,
      "num_tokens": 2484459.0,
      "reward": 2.6583719730377195,
      "reward_std": 0.41457981467247007,
      "rewards/classifier_reward": 0.9726575374603271,
      "rewards/length_reward": 0.6857142984867096,
      "rewards/slop_reward": 1.0,
      "step": 94
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 239.4571563720703,
      "epoch": 0.2375,
      "grad_norm": 8.130278831610969,
      "kl": 0.586328125,
      "learning_rate": 9.257645688666555e-07,
      "loss": 0.0006,
      "num_tokens": 2510647.0,
      "reward": 2.552411127090454,
      "reward_std": 0.444735050201416,
      "rewards/classifier_reward": 0.9220538377761841,
      "rewards/length_reward": 0.6571428656578064,
      "rewards/slop_reward": 0.9732142806053161,
      "step": 95
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 248.48572692871093,
      "epoch": 0.24,
      "grad_norm": 11.012318216722969,
      "kl": 0.559375,
      "learning_rate": 9.235233564979754e-07,
      "loss": 0.0006,
      "num_tokens": 2537166.0,
      "reward": 2.3741667747497557,
      "reward_std": 0.5231991052627564,
      "rewards/classifier_reward": 0.7955952703952789,
      "rewards/length_reward": 0.600000011920929,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 96
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 242.91429748535157,
      "epoch": 0.2425,
      "grad_norm": 9.607401037517901,
      "kl": 0.503125,
      "learning_rate": 9.212516110184794e-07,
      "loss": 0.0005,
      "num_tokens": 2563588.0,
      "reward": 2.6350881576538088,
      "reward_std": 0.47173853516578673,
      "rewards/classifier_reward": 0.8618737578392028,
      "rewards/length_reward": 0.8000000059604645,
      "rewards/slop_reward": 0.9732142806053161,
      "step": 97
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 193.971435546875,
      "epoch": 0.245,
      "grad_norm": 10.623433793787921,
      "kl": 0.7234375,
      "learning_rate": 9.189494962053368e-07,
      "loss": 0.0007,
      "num_tokens": 2588297.0,
      "reward": 2.389581322669983,
      "reward_std": 0.4812875479459763,
      "rewards/classifier_reward": 0.903866958618164,
      "rewards/length_reward": 0.4857142984867096,
      "rewards/slop_reward": 1.0,
      "step": 98
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 171.51429443359376,
      "epoch": 0.2475,
      "grad_norm": 11.438435269316447,
      "kl": 0.64375,
      "learning_rate": 9.166171780251364e-07,
      "loss": 0.0006,
      "num_tokens": 2612015.0,
      "reward": 2.513836717605591,
      "reward_std": 0.26052397638559344,
      "rewards/classifier_reward": 0.8852650642395019,
      "rewards/length_reward": 0.6285714328289032,
      "rewards/slop_reward": 1.0,
      "step": 99
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 167.14286346435546,
      "epoch": 0.25,
      "grad_norm": 11.041028322429328,
      "kl": 0.7359375,
      "learning_rate": 9.14254824621921e-07,
      "loss": 0.0007,
      "num_tokens": 2635785.0,
      "reward": 2.355943202972412,
      "reward_std": 0.33155601024627684,
      "rewards/classifier_reward": 0.8416573882102967,
      "rewards/length_reward": 0.5142857193946838,
      "rewards/slop_reward": 1.0,
      "step": 100
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 163.74286499023438,
      "epoch": 0.2525,
      "grad_norm": 9.00301084087431,
      "kl": 0.75390625,
      "learning_rate": 9.118626063050661e-07,
      "loss": 0.0012,
      "num_tokens": 2659436.0,
      "reward": 2.5006046295166016,
      "reward_std": 0.4252849280834198,
      "rewards/classifier_reward": 0.9506044745445251,
      "rewards/length_reward": 0.571428582072258,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 101
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 195.42858276367187,
      "epoch": 0.255,
      "grad_norm": 9.702866580028527,
      "kl": 0.79921875,
      "learning_rate": 9.094406955370008e-07,
      "loss": 0.0008,
      "num_tokens": 2683861.0,
      "reward": 2.5624767780303954,
      "reward_std": 0.47246721386909485,
      "rewards/classifier_reward": 0.9267622709274292,
      "rewards/length_reward": 0.6571428716182709,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 102
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 183.8857223510742,
      "epoch": 0.2575,
      "grad_norm": 7.999927126827247,
      "kl": 0.7359375,
      "learning_rate": 9.069892669207757e-07,
      "loss": 0.0007,
      "num_tokens": 2708217.0,
      "reward": 2.4374377012252806,
      "reward_std": 0.3513069462031126,
      "rewards/classifier_reward": 0.8374375879764557,
      "rewards/length_reward": 0.6000000059604644,
      "rewards/slop_reward": 1.0,
      "step": 103
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 203.74286804199218,
      "epoch": 0.26,
      "grad_norm": 8.535286841601573,
      "kl": 0.6046875,
      "learning_rate": 9.045084971874737e-07,
      "loss": 0.0006,
      "num_tokens": 2733039.0,
      "reward": 2.590514373779297,
      "reward_std": 0.3418044149875641,
      "rewards/classifier_reward": 0.7905142605304718,
      "rewards/length_reward": 0.8000000059604645,
      "rewards/slop_reward": 1.0,
      "step": 104
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 186.88572387695314,
      "epoch": 0.2625,
      "grad_norm": 11.150131541909593,
      "kl": 2.3078125,
      "learning_rate": 9.019985651834703e-07,
      "loss": 0.0023,
      "num_tokens": 2757500.0,
      "reward": 2.6002991676330565,
      "reward_std": 0.47049993872642515,
      "rewards/classifier_reward": 0.850299060344696,
      "rewards/length_reward": 0.7714285790920258,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 105
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 231.48572692871093,
      "epoch": 0.265,
      "grad_norm": 7.346740346808247,
      "kl": 0.694140625,
      "learning_rate": 8.994596518575391e-07,
      "loss": 0.0007,
      "num_tokens": 2783522.0,
      "reward": 2.6393914222717285,
      "reward_std": 0.4599771976470947,
      "rewards/classifier_reward": 0.8965341806411743,
      "rewards/length_reward": 0.7428571581840515,
      "rewards/slop_reward": 1.0,
      "step": 106
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 293.51429748535156,
      "epoch": 0.2675,
      "grad_norm": 7.782363877670758,
      "kl": 0.71328125,
      "learning_rate": 8.968919402478075e-07,
      "loss": 0.0007,
      "num_tokens": 2811715.0,
      "reward": 2.497778224945068,
      "reward_std": 0.49988613873720167,
      "rewards/classifier_reward": 0.8692066669464111,
      "rewards/length_reward": 0.6285714387893677,
      "rewards/slop_reward": 1.0,
      "step": 107
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 202.40000915527344,
      "epoch": 0.27,
      "grad_norm": 8.407812702495294,
      "kl": 0.9671875,
      "learning_rate": 8.942956154685595e-07,
      "loss": 0.001,
      "num_tokens": 2836377.0,
      "reward": 2.7594990730285645,
      "reward_std": 0.35219337940216067,
      "rewards/classifier_reward": 0.9237847208976746,
      "rewards/length_reward": 0.8571428656578064,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 108
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 311.14288024902345,
      "epoch": 0.2725,
      "grad_norm": 10.954001826900827,
      "kl": 0.803515625,
      "learning_rate": 8.916708646968923e-07,
      "loss": 0.0008,
      "num_tokens": 2865187.0,
      "reward": 2.3341631174087523,
      "reward_std": 0.3795748669654131,
      "rewards/classifier_reward": 0.6913058979436755,
      "rewards/length_reward": 0.6857142925262452,
      "rewards/slop_reward": 0.9571428537368775,
      "step": 109
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 295.88572387695314,
      "epoch": 0.275,
      "grad_norm": 7.24224369671047,
      "kl": 0.6296875,
      "learning_rate": 8.890178771592197e-07,
      "loss": 0.0006,
      "num_tokens": 2893081.0,
      "reward": 2.574794292449951,
      "reward_std": 0.4517861694097519,
      "rewards/classifier_reward": 0.9319370150566101,
      "rewards/length_reward": 0.6857142984867096,
      "rewards/slop_reward": 0.9571428537368775,
      "step": 110
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 310.4285858154297,
      "epoch": 0.2775,
      "grad_norm": 6.977209722960256,
      "kl": 1.34609375,
      "learning_rate": 8.863368441176325e-07,
      "loss": 0.0013,
      "num_tokens": 2921771.0,
      "reward": 2.2888038635253904,
      "reward_std": 0.6448704779148102,
      "rewards/classifier_reward": 0.8455002188682557,
      "rewards/length_reward": 0.5142857253551483,
      "rewards/slop_reward": 0.9290178537368774,
      "step": 111
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 251.82858276367188,
      "epoch": 0.28,
      "grad_norm": 6.8381698154250525,
      "kl": 0.86640625,
      "learning_rate": 8.836279588561081e-07,
      "loss": 0.0009,
      "num_tokens": 2948383.0,
      "reward": 2.4712666511535644,
      "reward_std": 0.4860814154148102,
      "rewards/classifier_reward": 0.8141236484050751,
      "rewards/length_reward": 0.6571428716182709,
      "rewards/slop_reward": 1.0,
      "step": 112
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 275.74287109375,
      "epoch": 0.2825,
      "grad_norm": 5.967234970457837,
      "kl": 0.8515625,
      "learning_rate": 8.808914166665772e-07,
      "loss": 0.0013,
      "num_tokens": 2975877.0,
      "reward": 2.6856667041778564,
      "reward_std": 0.38376912772655486,
      "rewards/classifier_reward": 0.9856665849685669,
      "rewards/length_reward": 0.7428571522235871,
      "rewards/slop_reward": 0.9571428537368775,
      "step": 113
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 291.1428680419922,
      "epoch": 0.285,
      "grad_norm": 6.696088875243213,
      "kl": 0.61484375,
      "learning_rate": 8.781274148348436e-07,
      "loss": 0.0006,
      "num_tokens": 3003901.0,
      "reward": 2.584765911102295,
      "reward_std": 0.441849821805954,
      "rewards/classifier_reward": 0.8704800248146057,
      "rewards/length_reward": 0.7142857313156128,
      "rewards/slop_reward": 1.0,
      "step": 114
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 286.6000152587891,
      "epoch": 0.2875,
      "grad_norm": 5.913822453334042,
      "kl": 0.678125,
      "learning_rate": 8.753361526263621e-07,
      "loss": 0.0007,
      "num_tokens": 3031852.0,
      "reward": 2.6028482913970947,
      "reward_std": 0.2958831213414669,
      "rewards/classifier_reward": 0.9957053542137146,
      "rewards/length_reward": 0.6285714387893677,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 115
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 292.20001525878905,
      "epoch": 0.29,
      "grad_norm": 5.738716860165764,
      "kl": 0.72578125,
      "learning_rate": 8.725178312718725e-07,
      "loss": 0.0012,
      "num_tokens": 3059999.0,
      "reward": 2.596400237083435,
      "reward_std": 0.3507813632488251,
      "rewards/classifier_reward": 0.953542971611023,
      "rewards/length_reward": 0.6857142925262452,
      "rewards/slop_reward": 0.9571428537368775,
      "step": 116
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 239.1428680419922,
      "epoch": 0.2925,
      "grad_norm": 69.40308583594562,
      "kl": 1.6,
      "learning_rate": 8.696726539528923e-07,
      "loss": 0.0021,
      "num_tokens": 3086289.0,
      "reward": 2.7766035079956053,
      "reward_std": 0.3328893929719925,
      "rewards/classifier_reward": 0.8908890843391418,
      "rewards/length_reward": 0.8857142925262451,
      "rewards/slop_reward": 1.0,
      "step": 117
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 259.22858276367185,
      "epoch": 0.295,
      "grad_norm": 5.40703806614853,
      "kl": 0.734765625,
      "learning_rate": 8.668008257870682e-07,
      "loss": 0.0012,
      "num_tokens": 3113282.0,
      "reward": 2.7762694358825684,
      "reward_std": 0.2866129666566849,
      "rewards/classifier_reward": 0.9476978421211243,
      "rewards/length_reward": 0.8285714328289032,
      "rewards/slop_reward": 1.0,
      "step": 118
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 201.91429443359374,
      "epoch": 0.2975,
      "grad_norm": 7.252699872821293,
      "kl": 0.81015625,
      "learning_rate": 8.639025538133897e-07,
      "loss": 0.0013,
      "num_tokens": 3138256.0,
      "reward": 2.851440095901489,
      "reward_std": 0.20830639004707335,
      "rewards/classifier_reward": 0.9085827589035034,
      "rewards/length_reward": 0.9428571462631226,
      "rewards/slop_reward": 1.0,
      "step": 119
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 250.20001525878905,
      "epoch": 0.3,
      "grad_norm": 6.062851209216701,
      "kl": 0.92578125,
      "learning_rate": 8.609780469772621e-07,
      "loss": 0.0014,
      "num_tokens": 3164933.0,
      "reward": 2.786311960220337,
      "reward_std": 0.29145972728729247,
      "rewards/classifier_reward": 0.9005975008010865,
      "rewards/length_reward": 0.8857142925262451,
      "rewards/slop_reward": 1.0,
      "step": 120
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 199.4857208251953,
      "epoch": 0.3025,
      "grad_norm": 8.363986536112053,
      "kl": 0.746875,
      "learning_rate": 8.580275161154431e-07,
      "loss": 0.0007,
      "num_tokens": 3189764.0,
      "reward": 2.89967794418335,
      "reward_std": 0.15721405297517776,
      "rewards/classifier_reward": 0.9496778607368469,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 121
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 218.88572387695314,
      "epoch": 0.305,
      "grad_norm": 104.91081313736798,
      "kl": 17.3171875,
      "learning_rate": 8.550511739408428e-07,
      "loss": 0.0182,
      "num_tokens": 3215345.0,
      "reward": 2.980521392822266,
      "reward_std": 0.05153606534004211,
      "rewards/classifier_reward": 0.9805211901664734,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 122
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 222.91430053710937,
      "epoch": 0.3075,
      "grad_norm": 7.509956960673518,
      "kl": 0.934375,
      "learning_rate": 8.520492350271895e-07,
      "loss": 0.001,
      "num_tokens": 3241067.0,
      "reward": 2.8307112216949464,
      "reward_std": 0.1734127746662125,
      "rewards/classifier_reward": 0.8521397054195404,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 123
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 240.17144165039062,
      "epoch": 0.31,
      "grad_norm": 6.959659369798759,
      "kl": 1.028125,
      "learning_rate": 8.490219157935588e-07,
      "loss": 0.0015,
      "num_tokens": 3267393.0,
      "reward": 2.7303539276123048,
      "reward_std": 0.1922714289277792,
      "rewards/classifier_reward": 0.9517823934555054,
      "rewards/length_reward": 0.8000000029802322,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 124
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 229.5428680419922,
      "epoch": 0.3125,
      "grad_norm": 8.221210886450612,
      "kl": 1.91640625,
      "learning_rate": 8.459694344887731e-07,
      "loss": 0.0019,
      "num_tokens": 3293186.0,
      "reward": 2.739912986755371,
      "reward_std": 0.36319895684719083,
      "rewards/classifier_reward": 0.9541985750198364,
      "rewards/length_reward": 0.8285714328289032,
      "rewards/slop_reward": 0.9571428537368775,
      "step": 125
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 216.02858276367186,
      "epoch": 0.315,
      "grad_norm": 6.737006006501442,
      "kl": 1.015625,
      "learning_rate": 8.428920111756657e-07,
      "loss": 0.0015,
      "num_tokens": 3318667.0,
      "reward": 2.7134992361068724,
      "reward_std": 0.2444542996585369,
      "rewards/classifier_reward": 0.8563561499118805,
      "rewards/length_reward": 0.8571428596973419,
      "rewards/slop_reward": 1.0,
      "step": 126
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 226.20001220703125,
      "epoch": 0.3175,
      "grad_norm": 4.36908357763648,
      "kl": 0.96640625,
      "learning_rate": 8.397898677152172e-07,
      "loss": 0.0024,
      "num_tokens": 3344503.0,
      "reward": 2.9305933475494386,
      "reward_std": 0.12042829990386963,
      "rewards/classifier_reward": 0.9591646075248719,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 1.0,
      "step": 127
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 203.31429443359374,
      "epoch": 0.32,
      "grad_norm": 6.1156992776971855,
      "kl": 0.85703125,
      "learning_rate": 8.366632277505597e-07,
      "loss": 0.0018,
      "num_tokens": 3369294.0,
      "reward": 2.9620502471923826,
      "reward_std": 0.07551092505455018,
      "rewards/classifier_reward": 0.983478581905365,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 128
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 220.0285858154297,
      "epoch": 0.3225,
      "grad_norm": 6.8222141729156975,
      "kl": 0.85859375,
      "learning_rate": 8.335123166908543e-07,
      "loss": 0.0013,
      "num_tokens": 3394915.0,
      "reward": 2.7929779529571532,
      "reward_std": 0.28287690281867983,
      "rewards/classifier_reward": 0.9340491890907288,
      "rewards/length_reward": 0.8857142865657807,
      "rewards/slop_reward": 0.9732142806053161,
      "step": 129
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 225.62857971191406,
      "epoch": 0.325,
      "grad_norm": 4.536469246397557,
      "kl": 1.11015625,
      "learning_rate": 8.303373616950406e-07,
      "loss": 0.0025,
      "num_tokens": 3420549.0,
      "reward": 2.9556642055511473,
      "reward_std": 0.07710518054664135,
      "rewards/classifier_reward": 0.9985211491584778,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 0.9571428537368775,
      "step": 130
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 220.68572387695312,
      "epoch": 0.3275,
      "grad_norm": 2.9694599664159407,
      "kl": 0.9546875,
      "learning_rate": 8.271385916554604e-07,
      "loss": 0.0029,
      "num_tokens": 3445788.0,
      "reward": 2.996094989776611,
      "reward_std": 0.010332237184047698,
      "rewards/classifier_reward": 0.9960947871208191,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 131
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 296.20001831054685,
      "epoch": 0.33,
      "grad_norm": 4.064839178436649,
      "kl": 1.1390625,
      "learning_rate": 8.23916237181355e-07,
      "loss": 0.0016,
      "num_tokens": 3474044.0,
      "reward": 2.4268852710723876,
      "reward_std": 0.15440489053726197,
      "rewards/classifier_reward": 0.9697423577308655,
      "rewards/length_reward": 0.4571428596973419,
      "rewards/slop_reward": 1.0,
      "step": 132
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 282.771435546875,
      "epoch": 0.3325,
      "grad_norm": 10.750678349276473,
      "kl": 0.82734375,
      "learning_rate": 8.206705305822412e-07,
      "loss": 0.0013,
      "num_tokens": 3501861.0,
      "reward": 2.4831544876098635,
      "reward_std": 0.26082203090190886,
      "rewards/classifier_reward": 0.9831543445587159,
      "rewards/length_reward": 0.5428571492433548,
      "rewards/slop_reward": 0.9571428537368775,
      "step": 133
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 247.34286804199218,
      "epoch": 0.335,
      "grad_norm": 106.5245827902456,
      "kl": 88.11328125,
      "learning_rate": 8.174017058511628e-07,
      "loss": 0.0893,
      "num_tokens": 3528356.0,
      "reward": 2.8546416759490967,
      "reward_std": 0.2578580856323242,
      "rewards/classifier_reward": 0.9903557300567627,
      "rewards/length_reward": 0.8857142925262451,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 134
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 280.2857299804688,
      "epoch": 0.3375,
      "grad_norm": 5.075379696681929,
      "kl": 1.60859375,
      "learning_rate": 8.141099986478212e-07,
      "loss": 0.0021,
      "num_tokens": 3555922.0,
      "reward": 2.635714387893677,
      "reward_std": 0.31418272852897644,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.6571428656578064,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 135
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 304.5714416503906,
      "epoch": 0.34,
      "grad_norm": 5.929984152853479,
      "kl": 1.1859375,
      "learning_rate": 8.107956462815861e-07,
      "loss": 0.0017,
      "num_tokens": 3584471.0,
      "reward": 2.3495986461639404,
      "reward_std": 0.3951677083969116,
      "rewards/classifier_reward": 0.8853127479553222,
      "rewards/length_reward": 0.4857142955064774,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 136
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 288.2857299804688,
      "epoch": 0.3425,
      "grad_norm": 8.1455480169376,
      "kl": 1.215625,
      "learning_rate": 8.074588876943872e-07,
      "loss": 0.0012,
      "num_tokens": 3612481.0,
      "reward": 2.4660715579986574,
      "reward_std": 0.5484442114830017,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.5142857283353806,
      "rewards/slop_reward": 0.9517857074737549,
      "step": 137
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 267.8000061035156,
      "epoch": 0.345,
      "grad_norm": 5.930202214193798,
      "kl": 0.98125,
      "learning_rate": 8.040999634434882e-07,
      "loss": 0.0015,
      "num_tokens": 3639774.0,
      "reward": 2.7785715579986574,
      "reward_std": 0.39145426750183104,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.800000011920929,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 138
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 255.57144165039062,
      "epoch": 0.3475,
      "grad_norm": 4.399939053025549,
      "kl": 1.53125,
      "learning_rate": 8.00719115684144e-07,
      "loss": 0.0025,
      "num_tokens": 3666639.0,
      "reward": 2.6642858505249025,
      "reward_std": 0.21827136874198913,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.6857142925262452,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 139
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 251.571435546875,
      "epoch": 0.35,
      "grad_norm": 6.704733780428727,
      "kl": 1.0859375,
      "learning_rate": 7.973165881521433e-07,
      "loss": 0.002,
      "num_tokens": 3693159.0,
      "reward": 2.8857144355773925,
      "reward_std": 0.24877579212188722,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.8857142925262451,
      "rewards/slop_reward": 1.0,
      "step": 140
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 248.9428680419922,
      "epoch": 0.3525,
      "grad_norm": 3.4261047442168397,
      "kl": 0.93359375,
      "learning_rate": 7.938926261462365e-07,
      "loss": 0.0028,
      "num_tokens": 3719792.0,
      "reward": 2.742554450035095,
      "reward_std": 0.16382334232330323,
      "rewards/classifier_reward": 0.8568399548530579,
      "rewards/length_reward": 0.8857142865657807,
      "rewards/slop_reward": 1.0,
      "step": 141
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 212.3714385986328,
      "epoch": 0.355,
      "grad_norm": 6.044238574227886,
      "kl": 1.3890625,
      "learning_rate": 7.90447476510452e-07,
      "loss": 0.0028,
      "num_tokens": 3745103.0,
      "reward": 2.9194665908813477,
      "reward_std": 0.21307192444801332,
      "rewards/classifier_reward": 0.9980378150939941,
      "rewards/length_reward": 0.9428571462631226,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 142
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 184.5428680419922,
      "epoch": 0.3575,
      "grad_norm": 4.2944694392303155,
      "kl": 1.140625,
      "learning_rate": 7.869813876162998e-07,
      "loss": 0.003,
      "num_tokens": 3769090.0,
      "reward": 2.9714287757873534,
      "reward_std": 0.07559289336204529,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 1.0,
      "step": 143
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 207.1428680419922,
      "epoch": 0.36,
      "grad_norm": 2.4809967074334627,
      "kl": 1.1765625,
      "learning_rate": 7.834946093448658e-07,
      "loss": 0.0031,
      "num_tokens": 3794079.0,
      "reward": 2.9714287757873534,
      "reward_std": 0.07559289336204529,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 1.0,
      "step": 144
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 197.17143859863282,
      "epoch": 0.3625,
      "grad_norm": 3.755882982422092,
      "kl": 1.396875,
      "learning_rate": 7.799873930687977e-07,
      "loss": 0.0033,
      "num_tokens": 3818773.0,
      "reward": 2.978571653366089,
      "reward_std": 0.056694668531417844,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 145
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 187.71429443359375,
      "epoch": 0.365,
      "grad_norm": 6.482025225350517,
      "kl": 2.0046875,
      "learning_rate": 7.764599916341816e-07,
      "loss": 0.003,
      "num_tokens": 3843103.0,
      "reward": 2.892857313156128,
      "reward_std": 0.22987756729125977,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9142857193946838,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 146
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 214.8857208251953,
      "epoch": 0.3675,
      "grad_norm": 3.8265113738201806,
      "kl": 1.3890625,
      "learning_rate": 7.729126593423149e-07,
      "loss": 0.0033,
      "num_tokens": 3868513.0,
      "reward": 2.997367763519287,
      "reward_std": 0.006964774429798126,
      "rewards/classifier_reward": 0.9973675608634949,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 147
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 184.1428649902344,
      "epoch": 0.37,
      "grad_norm": 18.75869388269992,
      "kl": 1.515625,
      "learning_rate": 7.693456519313719e-07,
      "loss": 0.0029,
      "num_tokens": 3892878.0,
      "reward": 2.851199245452881,
      "reward_std": 0.18016420006752015,
      "rewards/classifier_reward": 0.9869133591651916,
      "rewards/length_reward": 0.8857142865657807,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 148
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 193.6571502685547,
      "epoch": 0.3725,
      "grad_norm": 5.454996617575025,
      "kl": 1.684375,
      "learning_rate": 7.657592265579669e-07,
      "loss": 0.0031,
      "num_tokens": 3917511.0,
      "reward": 2.993258571624756,
      "reward_std": 0.01783668529242277,
      "rewards/classifier_reward": 0.9932583689689636,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 149
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 181.31429138183594,
      "epoch": 0.375,
      "grad_norm": 3.955698987070301,
      "kl": 1.3390625,
      "learning_rate": 7.621536417786158e-07,
      "loss": 0.0032,
      "num_tokens": 3941554.0,
      "reward": 2.9714287757873534,
      "reward_std": 0.07559289336204529,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 1.0,
      "step": 150
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 183.9714385986328,
      "epoch": 0.3775,
      "grad_norm": 4.5979098970153185,
      "kl": 1.528125,
      "learning_rate": 7.585291575310952e-07,
      "loss": 0.0034,
      "num_tokens": 3965818.0,
      "reward": 2.9714287757873534,
      "reward_std": 0.07559289336204529,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 1.0,
      "step": 151
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 175.71429443359375,
      "epoch": 0.38,
      "grad_norm": 7.501271856937242,
      "kl": 1.790625,
      "learning_rate": 7.548860351157027e-07,
      "loss": 0.0027,
      "num_tokens": 3989746.0,
      "reward": 2.7536909580230713,
      "reward_std": 0.3092236161231995,
      "rewards/classifier_reward": 0.9251193881034852,
      "rewards/length_reward": 0.8285714387893677,
      "rewards/slop_reward": 1.0,
      "step": 152
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 210.571435546875,
      "epoch": 0.3825,
      "grad_norm": 5.2866175605,
      "kl": 2.065625,
      "learning_rate": 7.512245371764196e-07,
      "loss": 0.0035,
      "num_tokens": 4015036.0,
      "reward": 2.9245490074157714,
      "reward_std": 0.12335940003395081,
      "rewards/classifier_reward": 0.9727631211280823,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 0.9517857074737549,
      "step": 153
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 187.771435546875,
      "epoch": 0.385,
      "grad_norm": 4.602093297361531,
      "kl": 2.203125,
      "learning_rate": 7.475449276819752e-07,
      "loss": 0.0041,
      "num_tokens": 4039528.0,
      "reward": 2.942857360839844,
      "reward_std": 0.09759000539779664,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9428571462631226,
      "rewards/slop_reward": 1.0,
      "step": 154
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 175.60000610351562,
      "epoch": 0.3875,
      "grad_norm": 6.76968523287945,
      "kl": 1.71875,
      "learning_rate": 7.438474719068173e-07,
      "loss": 0.0031,
      "num_tokens": 4063594.0,
      "reward": 2.914285898208618,
      "reward_std": 0.1731828987598419,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9142857193946838,
      "rewards/slop_reward": 1.0,
      "step": 155
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 181.31429443359374,
      "epoch": 0.39,
      "grad_norm": 3.9415238872059795,
      "kl": 3.5171875,
      "learning_rate": 7.401324364119871e-07,
      "loss": 0.0054,
      "num_tokens": 4087555.0,
      "reward": 2.9714287757873534,
      "reward_std": 0.07559289336204529,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 1.0,
      "step": 156
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 199.22858276367188,
      "epoch": 0.3925,
      "grad_norm": 0.6743528375242909,
      "kl": 1.6703125,
      "learning_rate": 7.364000890259023e-07,
      "loss": 0.0041,
      "num_tokens": 4112265.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 157
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 203.02857971191406,
      "epoch": 0.395,
      "grad_norm": 0.21235184389038209,
      "kl": 1.5953125,
      "learning_rate": 7.326506988250487e-07,
      "loss": 0.004,
      "num_tokens": 4137291.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 158
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 211.00000915527343,
      "epoch": 0.3975,
      "grad_norm": 0.7871274406295002,
      "kl": 1.9828125,
      "learning_rate": 7.288845361145812e-07,
      "loss": 0.0044,
      "num_tokens": 4162596.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 159
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 205.82857971191407,
      "epoch": 0.4,
      "grad_norm": 0.3126714050105658,
      "kl": 1.596875,
      "learning_rate": 7.251018724088366e-07,
      "loss": 0.004,
      "num_tokens": 4187048.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 160
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 197.42857666015624,
      "epoch": 0.4025,
      "grad_norm": 0.3330658614195518,
      "kl": 1.5875,
      "learning_rate": 7.213029804117603e-07,
      "loss": 0.004,
      "num_tokens": 4211839.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 161
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 182.74286499023438,
      "epoch": 0.405,
      "grad_norm": 0.09053574072038517,
      "kl": 1.2125,
      "learning_rate": 7.174881339972448e-07,
      "loss": 0.0036,
      "num_tokens": 4236155.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 162
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 189.971435546875,
      "epoch": 0.4075,
      "grad_norm": 3.922781639240598,
      "kl": 1.1859375,
      "learning_rate": 7.136576081893863e-07,
      "loss": 0.0031,
      "num_tokens": 4260724.0,
      "reward": 2.968118953704834,
      "reward_std": 0.07463454008102417,
      "rewards/classifier_reward": 0.9966901540756226,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 1.0,
      "step": 163
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 211.22857971191405,
      "epoch": 0.41,
      "grad_norm": 0.5831119895182555,
      "kl": 1.22265625,
      "learning_rate": 7.09811679142657e-07,
      "loss": 0.0036,
      "num_tokens": 4285850.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 164
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 203.71429748535155,
      "epoch": 0.4125,
      "grad_norm": 2.3433145018681825,
      "kl": 1.1546875,
      "learning_rate": 7.059506241219964e-07,
      "loss": 0.0031,
      "num_tokens": 4310900.0,
      "reward": 2.9714287757873534,
      "reward_std": 0.07559289336204529,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 1.0,
      "step": 165
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 200.91429443359374,
      "epoch": 0.415,
      "grad_norm": 0.18814570159680577,
      "kl": 1.1671875,
      "learning_rate": 7.02074721482822e-07,
      "loss": 0.0036,
      "num_tokens": 4335852.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 166
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 212.08572692871093,
      "epoch": 0.4175,
      "grad_norm": 3.9253886023797246,
      "kl": 1.08125,
      "learning_rate": 6.981842506509626e-07,
      "loss": 0.0025,
      "num_tokens": 4361111.0,
      "reward": 2.950000190734863,
      "reward_std": 0.13228756189346313,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 167
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 202.82857971191407,
      "epoch": 0.42,
      "grad_norm": 118.43288868650292,
      "kl": 36.20390625,
      "learning_rate": 6.942794921025126e-07,
      "loss": 0.0382,
      "num_tokens": 4386130.0,
      "reward": 2.9714287757873534,
      "reward_std": 0.07559289336204529,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 1.0,
      "step": 168
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 194.0571533203125,
      "epoch": 0.4225,
      "grad_norm": 0.07954465329541446,
      "kl": 1.03984375,
      "learning_rate": 6.903607273436127e-07,
      "loss": 0.0034,
      "num_tokens": 4410840.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 169
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 193.7714385986328,
      "epoch": 0.425,
      "grad_norm": 0.0486408492542655,
      "kl": 0.978125,
      "learning_rate": 6.864282388901543e-07,
      "loss": 0.0034,
      "num_tokens": 4435370.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 170
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 208.08572692871093,
      "epoch": 0.4275,
      "grad_norm": 2.6728463524577752,
      "kl": 0.92265625,
      "learning_rate": 6.824823102474126e-07,
      "loss": 0.0028,
      "num_tokens": 4460308.0,
      "reward": 2.942857360839844,
      "reward_std": 0.09759000539779664,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9428571462631226,
      "rewards/slop_reward": 1.0,
      "step": 171
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 202.22858276367188,
      "epoch": 0.43,
      "grad_norm": 0.13449796435482986,
      "kl": 1.034375,
      "learning_rate": 6.785232258896076e-07,
      "loss": 0.0034,
      "num_tokens": 4485226.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 172
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 182.68572082519532,
      "epoch": 0.4325,
      "grad_norm": 7.189835012946043,
      "kl": 0.95078125,
      "learning_rate": 6.745512712393957e-07,
      "loss": 0.0024,
      "num_tokens": 4509446.0,
      "reward": 2.950000190734863,
      "reward_std": 0.13228756189346313,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 173
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 192.20000915527345,
      "epoch": 0.435,
      "grad_norm": 3.314581838568204,
      "kl": 0.9203125,
      "learning_rate": 6.705667326472924e-07,
      "loss": 0.0028,
      "num_tokens": 4533638.0,
      "reward": 2.992787170410156,
      "reward_std": 0.019083873927593233,
      "rewards/classifier_reward": 0.992786979675293,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 174
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 181.7714416503906,
      "epoch": 0.4375,
      "grad_norm": 4.868845585454781,
      "kl": 0.91015625,
      "learning_rate": 6.665698973710288e-07,
      "loss": 0.0023,
      "num_tokens": 4557920.0,
      "reward": 2.914285898208618,
      "reward_std": 0.1731828987598419,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9142857193946838,
      "rewards/slop_reward": 1.0,
      "step": 175
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 202.4857208251953,
      "epoch": 0.44,
      "grad_norm": 5.277192051669204,
      "kl": 0.9015625,
      "learning_rate": 6.625610535548417e-07,
      "loss": 0.0028,
      "num_tokens": 4582927.0,
      "reward": 2.971197080612183,
      "reward_std": 0.07549313902854919,
      "rewards/classifier_reward": 0.9997682809829712,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 1.0,
      "step": 176
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 191.3714385986328,
      "epoch": 0.4425,
      "grad_norm": 3.5895293665237213,
      "kl": 0.86953125,
      "learning_rate": 6.58540490208701e-07,
      "loss": 0.0028,
      "num_tokens": 4607545.0,
      "reward": 2.9714287757873534,
      "reward_std": 0.07559289336204529,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 1.0,
      "step": 177
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 204.74286804199218,
      "epoch": 0.445,
      "grad_norm": 3.6924099151163348,
      "kl": 0.88046875,
      "learning_rate": 6.545084971874736e-07,
      "loss": 0.0023,
      "num_tokens": 4632631.0,
      "reward": 2.9700303077697754,
      "reward_std": 0.07929292395710945,
      "rewards/classifier_reward": 0.9986015200614929,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 1.0,
      "step": 178
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 210.74286804199218,
      "epoch": 0.4475,
      "grad_norm": 4.558365673779022,
      "kl": 1.03125,
      "learning_rate": 6.504653651700277e-07,
      "loss": 0.0025,
      "num_tokens": 4657813.0,
      "reward": 2.950000190734863,
      "reward_std": 0.13228756189346313,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 179
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 209.17143859863282,
      "epoch": 0.45,
      "grad_norm": 0.04420464498654459,
      "kl": 0.86640625,
      "learning_rate": 6.464113856382751e-07,
      "loss": 0.0033,
      "num_tokens": 4683054.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 180
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 218.4857208251953,
      "epoch": 0.4525,
      "grad_norm": 2.557396683384517,
      "kl": 1.0640625,
      "learning_rate": 6.423468508561598e-07,
      "loss": 0.003,
      "num_tokens": 4708257.0,
      "reward": 2.9714287757873534,
      "reward_std": 0.07559289336204529,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 1.0,
      "step": 181
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 229.88572387695314,
      "epoch": 0.455,
      "grad_norm": 3.1120823972576854,
      "kl": 1.415625,
      "learning_rate": 6.382720538485855e-07,
      "loss": 0.0033,
      "num_tokens": 4734223.0,
      "reward": 2.942857360839844,
      "reward_std": 0.09759000539779664,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9428571462631226,
      "rewards/slop_reward": 1.0,
      "step": 182
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 226.40000915527344,
      "epoch": 0.4575,
      "grad_norm": 0.06880695435363977,
      "kl": 0.846875,
      "learning_rate": 6.341872883802922e-07,
      "loss": 0.0032,
      "num_tokens": 4759812.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 183
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 235.05715637207032,
      "epoch": 0.46,
      "grad_norm": 2.2167569033565937,
      "kl": 0.75546875,
      "learning_rate": 6.300928489346765e-07,
      "loss": 0.0027,
      "num_tokens": 4785935.0,
      "reward": 2.9714287757873534,
      "reward_std": 0.07559289336204529,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 1.0,
      "step": 184
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 227.91429443359374,
      "epoch": 0.4625,
      "grad_norm": 2.5442698264193897,
      "kl": 0.7765625,
      "learning_rate": 6.259890306925626e-07,
      "loss": 0.0027,
      "num_tokens": 4811832.0,
      "reward": 2.9714287757873534,
      "reward_std": 0.07559289336204529,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 1.0,
      "step": 185
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 221.2571533203125,
      "epoch": 0.465,
      "grad_norm": 4.529163471508088,
      "kl": 0.79765625,
      "learning_rate": 6.218761295109208e-07,
      "loss": 0.0018,
      "num_tokens": 4837398.0,
      "reward": 2.9214287281036375,
      "reward_std": 0.2078804552555084,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9428571462631226,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 186
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 218.40000915527344,
      "epoch": 0.4675,
      "grad_norm": 0.04502419697939018,
      "kl": 0.8265625,
      "learning_rate": 6.177544419015387e-07,
      "loss": 0.0032,
      "num_tokens": 4862776.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 187
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 224.85715026855468,
      "epoch": 0.47,
      "grad_norm": 2.7870230542981322,
      "kl": 0.825,
      "learning_rate": 6.13624265009645e-07,
      "loss": 0.0027,
      "num_tokens": 4888566.0,
      "reward": 2.978571653366089,
      "reward_std": 0.056694668531417844,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 188
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 228.0571502685547,
      "epoch": 0.4725,
      "grad_norm": 0.06424356320573535,
      "kl": 0.8890625,
      "learning_rate": 6.094858965924866e-07,
      "loss": 0.0033,
      "num_tokens": 4914374.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 189
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 224.4571533203125,
      "epoch": 0.475,
      "grad_norm": 4.417906183651794,
      "kl": 0.90703125,
      "learning_rate": 6.053396349978631e-07,
      "loss": 0.0023,
      "num_tokens": 4940085.0,
      "reward": 2.8559008598327638,
      "reward_std": 0.20502071976661682,
      "rewards/classifier_reward": 0.9344720721244812,
      "rewards/length_reward": 0.9428571462631226,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 190
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 228.57143859863282,
      "epoch": 0.4775,
      "grad_norm": 0.7826472563807869,
      "kl": 0.95703125,
      "learning_rate": 6.011857791426178e-07,
      "loss": 0.0033,
      "num_tokens": 4966005.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 191
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 228.171435546875,
      "epoch": 0.48,
      "grad_norm": 0.2991397618360992,
      "kl": 1.06015625,
      "learning_rate": 5.970246284910876e-07,
      "loss": 0.0034,
      "num_tokens": 4991803.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 192
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 245.6285827636719,
      "epoch": 0.4825,
      "grad_norm": 4.590555458209977,
      "kl": 0.915625,
      "learning_rate": 5.92856483033514e-07,
      "loss": 0.0023,
      "num_tokens": 5018320.0,
      "reward": 2.950000190734863,
      "reward_std": 0.13228756189346313,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 193
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 231.00001220703126,
      "epoch": 0.485,
      "grad_norm": 3.8249513457346622,
      "kl": 0.83125,
      "learning_rate": 5.886816432644154e-07,
      "loss": 0.0023,
      "num_tokens": 5044075.0,
      "reward": 2.9297013759613035,
      "reward_std": 0.12975128293037413,
      "rewards/classifier_reward": 0.9797011494636536,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 194
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 232.00000915527343,
      "epoch": 0.4875,
      "grad_norm": 3.4135057358827834,
      "kl": 1.3140625,
      "learning_rate": 5.845004101609246e-07,
      "loss": 0.0032,
      "num_tokens": 5069796.0,
      "reward": 2.9581347465515138,
      "reward_std": 0.10335763692855834,
      "rewards/classifier_reward": 0.9867059469223023,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 1.0,
      "step": 195
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 236.68572692871095,
      "epoch": 0.49,
      "grad_norm": 4.013642506936478,
      "kl": 0.85390625,
      "learning_rate": 5.803130851610885e-07,
      "loss": 0.0023,
      "num_tokens": 5095958.0,
      "reward": 2.9428573131561278,
      "reward_std": 0.15118578672409058,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9428571462631226,
      "rewards/slop_reward": 1.0,
      "step": 196
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 238.4571533203125,
      "epoch": 0.4925,
      "grad_norm": 4.603644905524406,
      "kl": 0.940625,
      "learning_rate": 5.761199701421391e-07,
      "loss": 0.0019,
      "num_tokens": 5121931.0,
      "reward": 2.941946840286255,
      "reward_std": 0.15359463561326264,
      "rewards/classifier_reward": 0.9990895390510559,
      "rewards/length_reward": 0.9428571462631226,
      "rewards/slop_reward": 1.0,
      "step": 197
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 238.02857971191406,
      "epoch": 0.495,
      "grad_norm": 2.4430881235820507,
      "kl": 0.88046875,
      "learning_rate": 5.719213673987276e-07,
      "loss": 0.0028,
      "num_tokens": 5148140.0,
      "reward": 2.9714287757873534,
      "reward_std": 0.07559289336204529,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 1.0,
      "step": 198
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 228.7714385986328,
      "epoch": 0.4975,
      "grad_norm": 2.885447633946335,
      "kl": 0.8984375,
      "learning_rate": 5.677175796211332e-07,
      "loss": 0.0028,
      "num_tokens": 5173797.0,
      "reward": 2.978571653366089,
      "reward_std": 0.056694668531417844,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 199
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 234.77144470214844,
      "epoch": 0.5,
      "grad_norm": 0.04186680424978939,
      "kl": 0.9296875,
      "learning_rate": 5.635089098734393e-07,
      "loss": 0.0033,
      "num_tokens": 5199798.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 200
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 224.51429443359376,
      "epoch": 0.5025,
      "grad_norm": 2.794225657350946,
      "kl": 0.95,
      "learning_rate": 5.592956615716866e-07,
      "loss": 0.0029,
      "num_tokens": 5225576.0,
      "reward": 2.978571653366089,
      "reward_std": 0.056694668531417844,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 201
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 216.02857971191406,
      "epoch": 0.505,
      "grad_norm": 4.294508436959706,
      "kl": 1.10625,
      "learning_rate": 5.550781384619973e-07,
      "loss": 0.0025,
      "num_tokens": 5251038.0,
      "reward": 2.975967788696289,
      "reward_std": 0.0635837346315384,
      "rewards/classifier_reward": 0.9759676098823548,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 202
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 218.571435546875,
      "epoch": 0.5075,
      "grad_norm": 0.043507163632059365,
      "kl": 0.91484375,
      "learning_rate": 5.50856644598678e-07,
      "loss": 0.0033,
      "num_tokens": 5276396.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 203
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 205.6571502685547,
      "epoch": 0.51,
      "grad_norm": 0.09961196396275367,
      "kl": 1.0578125,
      "learning_rate": 5.466314843222993e-07,
      "loss": 0.0034,
      "num_tokens": 5301460.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 204
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 225.9714385986328,
      "epoch": 0.5125,
      "grad_norm": 3.605683444813084,
      "kl": 0.9609375,
      "learning_rate": 5.424029622377546e-07,
      "loss": 0.0029,
      "num_tokens": 5327289.0,
      "reward": 2.978571653366089,
      "reward_std": 0.056694668531417844,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 205
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 215.80001220703124,
      "epoch": 0.515,
      "grad_norm": 0.19828091837058953,
      "kl": 1.23828125,
      "learning_rate": 5.381713831923007e-07,
      "loss": 0.0036,
      "num_tokens": 5352596.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 206
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 214.8571563720703,
      "epoch": 0.5175,
      "grad_norm": 2.9547607246974565,
      "kl": 1.12890625,
      "learning_rate": 5.339370522535804e-07,
      "loss": 0.003,
      "num_tokens": 5377938.0,
      "reward": 2.992868709564209,
      "reward_std": 0.018868234753608704,
      "rewards/classifier_reward": 0.9928684830665588,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 207
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 252.00001525878906,
      "epoch": 0.52,
      "grad_norm": 871.3268746641542,
      "kl": 64.790625,
      "learning_rate": 5.297002746876284e-07,
      "loss": 0.0667,
      "num_tokens": 5404678.0,
      "reward": 2.942857360839844,
      "reward_std": 0.09759000539779664,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9428571462631226,
      "rewards/slop_reward": 1.0,
      "step": 208
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 271.0857269287109,
      "epoch": 0.5225,
      "grad_norm": 13961.922953195786,
      "kl": 6094.1890625,
      "learning_rate": 5.254613559368648e-07,
      "loss": 6.1111,
      "num_tokens": 5432086.0,
      "reward": 2.8857144832611086,
      "reward_std": 0.18249738812446595,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.8857142925262451,
      "rewards/slop_reward": 1.0,
      "step": 209
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 257.9714385986328,
      "epoch": 0.525,
      "grad_norm": 8.061516897407943,
      "kl": 1.028125,
      "learning_rate": 5.212206015980741e-07,
      "loss": 0.0025,
      "num_tokens": 5459016.0,
      "reward": 2.828571605682373,
      "reward_std": 0.1731828987598419,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.8285714328289032,
      "rewards/slop_reward": 1.0,
      "step": 210
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 234.3714385986328,
      "epoch": 0.5275,
      "grad_norm": 0.05294394256827684,
      "kl": 1.0125,
      "learning_rate": 5.169783174003744e-07,
      "loss": 0.0034,
      "num_tokens": 5484886.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 211
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 244.20001220703125,
      "epoch": 0.53,
      "grad_norm": 2.7195604219419645,
      "kl": 1.353125,
      "learning_rate": 5.127348091831755e-07,
      "loss": 0.0033,
      "num_tokens": 5511353.0,
      "reward": 2.9714287757873534,
      "reward_std": 0.07559289336204529,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 1.0,
      "step": 212
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 247.80001220703124,
      "epoch": 0.5325,
      "grad_norm": 2.435046990313359,
      "kl": 0.98671875,
      "learning_rate": 5.084903828741312e-07,
      "loss": 0.0029,
      "num_tokens": 5537879.0,
      "reward": 2.828571653366089,
      "reward_std": 0.07559289336204529,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.8285714298486709,
      "rewards/slop_reward": 1.0,
      "step": 213
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 233.82858276367188,
      "epoch": 0.535,
      "grad_norm": 2.848539339582319,
      "kl": 1.053125,
      "learning_rate": 5.042453444670828e-07,
      "loss": 0.003,
      "num_tokens": 5563937.0,
      "reward": 2.9500002384185793,
      "reward_std": 0.0866025447845459,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 214
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 243.6285827636719,
      "epoch": 0.5375,
      "grad_norm": 4.574390608580604,
      "kl": 0.91953125,
      "learning_rate": 5e-07,
      "loss": 0.0024,
      "num_tokens": 5590384.0,
      "reward": 2.8857144832611086,
      "reward_std": 0.19518001079559327,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.8857142925262451,
      "rewards/slop_reward": 1.0,
      "step": 215
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 284.0571563720703,
      "epoch": 0.54,
      "grad_norm": 5.393964971542159,
      "kl": 1.16484375,
      "learning_rate": 4.957546555329173e-07,
      "loss": 0.0016,
      "num_tokens": 5618238.0,
      "reward": 2.6285715103149414,
      "reward_std": 0.25809029340744016,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.6285714328289032,
      "rewards/slop_reward": 1.0,
      "step": 216
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 244.40001525878907,
      "epoch": 0.5425,
      "grad_norm": 3.809477712898502,
      "kl": 0.7828125,
      "learning_rate": 4.915096171258689e-07,
      "loss": 0.0022,
      "num_tokens": 5644712.0,
      "reward": 2.828571605682373,
      "reward_std": 0.17318291068077088,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.8285714328289032,
      "rewards/slop_reward": 1.0,
      "step": 217
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 227.60001220703126,
      "epoch": 0.545,
      "grad_norm": 0.7163836490261896,
      "kl": 1.0765625,
      "learning_rate": 4.872651908168244e-07,
      "loss": 0.0035,
      "num_tokens": 5670466.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 218
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 210.7714416503906,
      "epoch": 0.5475,
      "grad_norm": 0.06375443345853048,
      "kl": 0.86953125,
      "learning_rate": 4.830216825996256e-07,
      "loss": 0.0033,
      "num_tokens": 5695540.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 219
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 212.02858276367186,
      "epoch": 0.55,
      "grad_norm": 0.044596037608717574,
      "kl": 0.8765625,
      "learning_rate": 4.787793984019259e-07,
      "loss": 0.0033,
      "num_tokens": 5720881.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 220
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 200.80000915527344,
      "epoch": 0.5525,
      "grad_norm": 0.16974934473523362,
      "kl": 1.16640625,
      "learning_rate": 4.7453864406313536e-07,
      "loss": 0.0036,
      "num_tokens": 5745792.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 221
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 211.22857971191405,
      "epoch": 0.555,
      "grad_norm": 0.0818904708032968,
      "kl": 0.9921875,
      "learning_rate": 4.7029972531237154e-07,
      "loss": 0.0034,
      "num_tokens": 5770873.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 222
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 195.2571533203125,
      "epoch": 0.5575,
      "grad_norm": 0.05116820241495938,
      "kl": 0.9,
      "learning_rate": 4.6606294774641965e-07,
      "loss": 0.0033,
      "num_tokens": 5795571.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 223
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 190.57143859863282,
      "epoch": 0.56,
      "grad_norm": 3.069339312788395,
      "kl": 0.9046875,
      "learning_rate": 4.6182861680769923e-07,
      "loss": 0.0028,
      "num_tokens": 5819962.0,
      "reward": 2.9714287757873534,
      "reward_std": 0.07559289336204529,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 1.0,
      "step": 224
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 195.08572387695312,
      "epoch": 0.5625,
      "grad_norm": 2.051249437321434,
      "kl": 0.890625,
      "learning_rate": 4.5759703776224555e-07,
      "loss": 0.0028,
      "num_tokens": 5844710.0,
      "reward": 2.954781198501587,
      "reward_std": 0.0808977723121643,
      "rewards/classifier_reward": 0.9833523750305175,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 1.0,
      "step": 225
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 194.11429748535156,
      "epoch": 0.565,
      "grad_norm": 0.06947745807380493,
      "kl": 0.915625,
      "learning_rate": 4.5336851567770074e-07,
      "loss": 0.0033,
      "num_tokens": 5869322.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 226
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 196.68572082519532,
      "epoch": 0.5675,
      "grad_norm": 3.6616246314488534,
      "kl": 0.75234375,
      "learning_rate": 4.4914335540132204e-07,
      "loss": 0.0027,
      "num_tokens": 5893903.0,
      "reward": 2.9714287757873534,
      "reward_std": 0.07559289336204529,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 1.0,
      "step": 227
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 190.91429443359374,
      "epoch": 0.57,
      "grad_norm": 2.9722209992367854,
      "kl": 2.09140625,
      "learning_rate": 4.4492186153800284e-07,
      "loss": 0.004,
      "num_tokens": 5918505.0,
      "reward": 2.942857360839844,
      "reward_std": 0.09759000539779664,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9428571462631226,
      "rewards/slop_reward": 1.0,
      "step": 228
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 184.91429138183594,
      "epoch": 0.5725,
      "grad_norm": 0.06042948982632837,
      "kl": 0.8359375,
      "learning_rate": 4.407043384283136e-07,
      "loss": 0.0032,
      "num_tokens": 5942897.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 229
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 182.25715026855468,
      "epoch": 0.575,
      "grad_norm": 0.07756768631242807,
      "kl": 0.8734375,
      "learning_rate": 4.364910901265606e-07,
      "loss": 0.0033,
      "num_tokens": 5967196.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 230
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 173.02857971191406,
      "epoch": 0.5775,
      "grad_norm": 0.9892808129257086,
      "kl": 1.546875,
      "learning_rate": 4.3228242037886687e-07,
      "loss": 0.0039,
      "num_tokens": 5990830.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 231
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 162.94286499023437,
      "epoch": 0.58,
      "grad_norm": 0.0578879268472676,
      "kl": 0.97578125,
      "learning_rate": 4.280786326012723e-07,
      "loss": 0.0034,
      "num_tokens": 6014434.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 232
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 165.42857666015624,
      "epoch": 0.5825,
      "grad_norm": 3.3348841449857387,
      "kl": 1.1046875,
      "learning_rate": 4.23880029857861e-07,
      "loss": 0.003,
      "num_tokens": 6038089.0,
      "reward": 2.914285945892334,
      "reward_std": 0.10690449476242066,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9142857193946838,
      "rewards/slop_reward": 1.0,
      "step": 233
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 163.71429138183595,
      "epoch": 0.585,
      "grad_norm": 3.485368879973419,
      "kl": 1.05703125,
      "learning_rate": 4.1968691483891133e-07,
      "loss": 0.003,
      "num_tokens": 6061739.0,
      "reward": 2.9714287757873534,
      "reward_std": 0.07559289336204529,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 1.0,
      "step": 234
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 163.00000915527343,
      "epoch": 0.5875,
      "grad_norm": 5.36230481850721,
      "kl": 1.00703125,
      "learning_rate": 4.154995898390755e-07,
      "loss": 0.002,
      "num_tokens": 6085364.0,
      "reward": 2.9397803783416747,
      "reward_std": 0.15932661443948745,
      "rewards/classifier_reward": 0.9969230651855469,
      "rewards/length_reward": 0.9428571462631226,
      "rewards/slop_reward": 1.0,
      "step": 235
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 177.0571502685547,
      "epoch": 0.59,
      "grad_norm": 3.3342971599613382,
      "kl": 1.04140625,
      "learning_rate": 4.1131835673558456e-07,
      "loss": 0.003,
      "num_tokens": 6109257.0,
      "reward": 2.9714287757873534,
      "reward_std": 0.07559289336204529,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 1.0,
      "step": 236
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 191.42857971191407,
      "epoch": 0.5925,
      "grad_norm": 0.08376084072008337,
      "kl": 1.02109375,
      "learning_rate": 4.0714351696648606e-07,
      "loss": 0.0034,
      "num_tokens": 6133846.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 237
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 192.08572692871093,
      "epoch": 0.595,
      "grad_norm": 0.046627278932137715,
      "kl": 0.98515625,
      "learning_rate": 4.029753715089123e-07,
      "loss": 0.0034,
      "num_tokens": 6158489.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 238
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 198.08572387695312,
      "epoch": 0.5975,
      "grad_norm": 11.565806887201926,
      "kl": 13.7140625,
      "learning_rate": 3.988142208573822e-07,
      "loss": 0.0161,
      "num_tokens": 6183159.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 239
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 197.31429443359374,
      "epoch": 0.6,
      "grad_norm": 0.05311856298193549,
      "kl": 1.02265625,
      "learning_rate": 3.94660365002137e-07,
      "loss": 0.0034,
      "num_tokens": 6207985.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 240
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 202.25715026855468,
      "epoch": 0.6025,
      "grad_norm": 4.782436637657736,
      "kl": 1.04921875,
      "learning_rate": 3.9051410340751346e-07,
      "loss": 0.0025,
      "num_tokens": 6232984.0,
      "reward": 2.8857144832611086,
      "reward_std": 0.1824974000453949,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.8857142925262451,
      "rewards/slop_reward": 1.0,
      "step": 241
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 193.6285827636719,
      "epoch": 0.605,
      "grad_norm": 3.232063907518827,
      "kl": 1.15859375,
      "learning_rate": 3.8637573499035503e-07,
      "loss": 0.0031,
      "num_tokens": 6257629.0,
      "reward": 2.879447841644287,
      "reward_std": 0.11382801532745361,
      "rewards/classifier_reward": 0.9937332987785339,
      "rewards/length_reward": 0.8857142865657807,
      "rewards/slop_reward": 1.0,
      "step": 242
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 195.4571502685547,
      "epoch": 0.6075,
      "grad_norm": 0.06471530202926354,
      "kl": 1.05625,
      "learning_rate": 3.822455580984613e-07,
      "loss": 0.0034,
      "num_tokens": 6282207.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 243
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 274.0571594238281,
      "epoch": 0.61,
      "grad_norm": 3.0795183504620107,
      "kl": 1.50390625,
      "learning_rate": 3.781238704890792e-07,
      "loss": 0.0034,
      "num_tokens": 6309000.0,
      "reward": 2.783582401275635,
      "reward_std": 0.023816290497779845,
      "rewards/classifier_reward": 0.9835822105407714,
      "rewards/length_reward": 0.8,
      "rewards/slop_reward": 1.0,
      "step": 244
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 210.28572692871094,
      "epoch": 0.6125,
      "grad_norm": 3.0694884073186777,
      "kl": 1.0015625,
      "learning_rate": 3.7401096930743746e-07,
      "loss": 0.0029,
      "num_tokens": 6334093.0,
      "reward": 2.9978450298309327,
      "reward_std": 0.005702095478773117,
      "rewards/classifier_reward": 0.9978448033332825,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 245
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 212.02857971191406,
      "epoch": 0.615,
      "grad_norm": 2.5150561307413764,
      "kl": 1.084375,
      "learning_rate": 3.699071510653235e-07,
      "loss": 0.003,
      "num_tokens": 6359434.0,
      "reward": 2.978571653366089,
      "reward_std": 0.056694668531417844,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 246
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 220.0571533203125,
      "epoch": 0.6175,
      "grad_norm": 0.04173992686860701,
      "kl": 0.96875,
      "learning_rate": 3.6581271161970784e-07,
      "loss": 0.0034,
      "num_tokens": 6384975.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 247
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 221.91429443359374,
      "epoch": 0.62,
      "grad_norm": 0.11609659945057246,
      "kl": 1.05078125,
      "learning_rate": 3.6172794615141446e-07,
      "loss": 0.0034,
      "num_tokens": 6410642.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 248
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 229.8571533203125,
      "epoch": 0.6225,
      "grad_norm": 0.08717380831309861,
      "kl": 0.971875,
      "learning_rate": 3.5765314914384024e-07,
      "loss": 0.0034,
      "num_tokens": 6436607.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 249
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 235.22858276367188,
      "epoch": 0.625,
      "grad_norm": 0.06146613518625495,
      "kl": 0.99765625,
      "learning_rate": 3.535886143617248e-07,
      "loss": 0.0034,
      "num_tokens": 6462760.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 250
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 238.17144165039062,
      "epoch": 0.6275,
      "grad_norm": 3.705436703551293,
      "kl": 1.01953125,
      "learning_rate": 3.495346348299724e-07,
      "loss": 0.0025,
      "num_tokens": 6488563.0,
      "reward": 2.9139774799346925,
      "reward_std": 0.1073996058665216,
      "rewards/classifier_reward": 0.9996915578842163,
      "rewards/length_reward": 0.9142857193946838,
      "rewards/slop_reward": 1.0,
      "step": 251
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 240.9714385986328,
      "epoch": 0.63,
      "grad_norm": 0.04691998567992384,
      "kl": 0.97890625,
      "learning_rate": 3.454915028125263e-07,
      "loss": 0.0034,
      "num_tokens": 6514734.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 252
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 228.02857971191406,
      "epoch": 0.6325,
      "grad_norm": 2.988060627858761,
      "kl": 1.0296875,
      "learning_rate": 3.4145950979129914e-07,
      "loss": 0.0029,
      "num_tokens": 6540498.0,
      "reward": 2.99902081489563,
      "reward_std": 0.0025911811739206315,
      "rewards/classifier_reward": 0.9990206360816956,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 253
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 218.54286499023436,
      "epoch": 0.635,
      "grad_norm": 0.09098624238326748,
      "kl": 1.11640625,
      "learning_rate": 3.3743894644515824e-07,
      "loss": 0.0035,
      "num_tokens": 6565616.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 254
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 223.57143859863282,
      "epoch": 0.6375,
      "grad_norm": 0.07923298438659351,
      "kl": 0.95703125,
      "learning_rate": 3.334301026289712e-07,
      "loss": 0.0033,
      "num_tokens": 6591361.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 255
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 219.00000610351563,
      "epoch": 0.64,
      "grad_norm": 3.44595488232409,
      "kl": 1.071875,
      "learning_rate": 3.294332673527076e-07,
      "loss": 0.003,
      "num_tokens": 6616850.0,
      "reward": 2.9714287757873534,
      "reward_std": 0.07559289336204529,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 1.0,
      "step": 256
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 220.80000915527344,
      "epoch": 0.6425,
      "grad_norm": 0.0704184343563711,
      "kl": 1.046875,
      "learning_rate": 3.254487287606044e-07,
      "loss": 0.0034,
      "num_tokens": 6642498.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 257
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 235.11429748535156,
      "epoch": 0.645,
      "grad_norm": 4.4341498316331425,
      "kl": 0.9890625,
      "learning_rate": 3.214767741103923e-07,
      "loss": 0.0024,
      "num_tokens": 6668511.0,
      "reward": 2.950000190734863,
      "reward_std": 0.13228756189346313,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 258
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 227.11429443359376,
      "epoch": 0.6475,
      "grad_norm": 2.667970216501767,
      "kl": 1.121875,
      "learning_rate": 3.1751768975258743e-07,
      "loss": 0.003,
      "num_tokens": 6694380.0,
      "reward": 2.9997310638427734,
      "reward_std": 0.0007120789494365453,
      "rewards/classifier_reward": 0.9997308611869812,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 259
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 224.42858276367187,
      "epoch": 0.65,
      "grad_norm": 0.049266434577523735,
      "kl": 1.04921875,
      "learning_rate": 3.135717611098457e-07,
      "loss": 0.0034,
      "num_tokens": 6719910.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 260
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 220.97144165039063,
      "epoch": 0.6525,
      "grad_norm": 0.046460428834944396,
      "kl": 1.00546875,
      "learning_rate": 3.0963927265638734e-07,
      "loss": 0.0034,
      "num_tokens": 6745328.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 261
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 229.2571533203125,
      "epoch": 0.655,
      "grad_norm": 2.6813194889359324,
      "kl": 1.08125,
      "learning_rate": 3.0572050789748726e-07,
      "loss": 0.003,
      "num_tokens": 6771231.0,
      "reward": 2.978571653366089,
      "reward_std": 0.056694668531417844,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 262
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 217.20001220703125,
      "epoch": 0.6575,
      "grad_norm": 2.8890479947848235,
      "kl": 1.16796875,
      "learning_rate": 3.018157493490374e-07,
      "loss": 0.0031,
      "num_tokens": 6796753.0,
      "reward": 2.978571653366089,
      "reward_std": 0.056694668531417844,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 263
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 233.51430053710936,
      "epoch": 0.66,
      "grad_norm": 2.802128759307218,
      "kl": 1.371875,
      "learning_rate": 2.9792527851717803e-07,
      "loss": 0.0033,
      "num_tokens": 6822476.0,
      "reward": 2.9714287757873534,
      "reward_std": 0.07559289336204529,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 1.0,
      "step": 264
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 225.82858581542968,
      "epoch": 0.6625,
      "grad_norm": 0.06018626969663984,
      "kl": 1.04296875,
      "learning_rate": 2.940493758780037e-07,
      "loss": 0.0034,
      "num_tokens": 6847958.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 265
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 214.571435546875,
      "epoch": 0.665,
      "grad_norm": 57.99972496539001,
      "kl": 2.54609375,
      "learning_rate": 2.9018832085734295e-07,
      "loss": 0.0045,
      "num_tokens": 6873054.0,
      "reward": 2.8571430683135985,
      "reward_std": 0.09759000539779664,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.8571428596973419,
      "rewards/slop_reward": 1.0,
      "step": 266
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 222.40001220703124,
      "epoch": 0.6675,
      "grad_norm": 5.391908835054465,
      "kl": 1.19921875,
      "learning_rate": 2.863423918106138e-07,
      "loss": 0.0022,
      "num_tokens": 6898757.0,
      "reward": 2.9923308849334718,
      "reward_std": 0.02029096046462655,
      "rewards/classifier_reward": 0.9923307299613953,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 267
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 226.80001220703124,
      "epoch": 0.67,
      "grad_norm": 0.44063981643559547,
      "kl": 1.27265625,
      "learning_rate": 2.825118660027553e-07,
      "loss": 0.0037,
      "num_tokens": 6924550.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 268
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 228.80001220703124,
      "epoch": 0.6725,
      "grad_norm": 3.710195212366325,
      "kl": 1.1890625,
      "learning_rate": 2.786970195882398e-07,
      "loss": 0.0026,
      "num_tokens": 6950478.0,
      "reward": 2.9779660224914553,
      "reward_std": 0.058296956680715085,
      "rewards/classifier_reward": 0.9993943929672241,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 269
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 227.51430053710936,
      "epoch": 0.675,
      "grad_norm": 99.1686641289829,
      "kl": 79.6328125,
      "learning_rate": 2.748981275911633e-07,
      "loss": 0.0819,
      "num_tokens": 6976266.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 270
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 242.82857971191407,
      "epoch": 0.6775,
      "grad_norm": 4.778004765539873,
      "kl": 1.34140625,
      "learning_rate": 2.7111546388541896e-07,
      "loss": 0.0028,
      "num_tokens": 7002514.0,
      "reward": 2.9389439105987547,
      "reward_std": 0.10570754185318947,
      "rewards/classifier_reward": 0.996086585521698,
      "rewards/length_reward": 0.9428571462631226,
      "rewards/slop_reward": 1.0,
      "step": 271
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 255.74286499023438,
      "epoch": 0.68,
      "grad_norm": 6.749318308021253,
      "kl": 1.54609375,
      "learning_rate": 2.673493011749513e-07,
      "loss": 0.003,
      "num_tokens": 7029221.0,
      "reward": 2.6523685693740844,
      "reward_std": 0.242851722240448,
      "rewards/classifier_reward": 0.9095112562179566,
      "rewards/length_reward": 0.7428571462631226,
      "rewards/slop_reward": 1.0,
      "step": 272
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 238.02857971191406,
      "epoch": 0.6825,
      "grad_norm": 4.816456021644019,
      "kl": 1.69375,
      "learning_rate": 2.635999109740976e-07,
      "loss": 0.0027,
      "num_tokens": 7055228.0,
      "reward": 2.858464765548706,
      "reward_std": 0.17900042831897736,
      "rewards/classifier_reward": 0.9941788673400879,
      "rewards/length_reward": 0.8857142865657807,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 273
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 241.7714385986328,
      "epoch": 0.685,
      "grad_norm": 2.4842492940641647,
      "kl": 1.0171875,
      "learning_rate": 2.598675635880129e-07,
      "loss": 0.0029,
      "num_tokens": 7081610.0,
      "reward": 2.9714287757873534,
      "reward_std": 0.07559289336204529,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 1.0,
      "step": 274
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 228.571435546875,
      "epoch": 0.6875,
      "grad_norm": 2.556011928350466,
      "kl": 1.15546875,
      "learning_rate": 2.561525280931828e-07,
      "loss": 0.0031,
      "num_tokens": 7107408.0,
      "reward": 2.978571653366089,
      "reward_std": 0.056694668531417844,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 275
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 231.6285827636719,
      "epoch": 0.69,
      "grad_norm": 0.08355624448786589,
      "kl": 1.053125,
      "learning_rate": 2.5245507231802486e-07,
      "loss": 0.0034,
      "num_tokens": 7133271.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 276
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 235.5428680419922,
      "epoch": 0.6925,
      "grad_norm": 3.2025322276348107,
      "kl": 1.4359375,
      "learning_rate": 2.487754628235805e-07,
      "loss": 0.0033,
      "num_tokens": 7159353.0,
      "reward": 2.9962107658386232,
      "reward_std": 0.01002594456076622,
      "rewards/classifier_reward": 0.9962105512619018,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 277
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 228.65715942382812,
      "epoch": 0.695,
      "grad_norm": 0.18581262102933002,
      "kl": 1.25859375,
      "learning_rate": 2.4511396488429724e-07,
      "loss": 0.0036,
      "num_tokens": 7185072.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 278
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 229.51429748535156,
      "epoch": 0.6975,
      "grad_norm": 4.453521381818444,
      "kl": 0.99765625,
      "learning_rate": 2.414708424689048e-07,
      "loss": 0.0024,
      "num_tokens": 7210683.0,
      "reward": 2.882569408416748,
      "reward_std": 0.2035010576248169,
      "rewards/classifier_reward": 0.9397120952606202,
      "rewards/length_reward": 0.9428571462631226,
      "rewards/slop_reward": 1.0,
      "step": 279
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 223.31429443359374,
      "epoch": 0.7,
      "grad_norm": 0.1121570381593698,
      "kl": 1.2296875,
      "learning_rate": 2.378463582213842e-07,
      "loss": 0.0036,
      "num_tokens": 7236399.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 280
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 217.60000915527343,
      "epoch": 0.7025,
      "grad_norm": 4.495034027794915,
      "kl": 0.9515625,
      "learning_rate": 2.3424077344203307e-07,
      "loss": 0.0024,
      "num_tokens": 7261935.0,
      "reward": 2.99290018081665,
      "reward_std": 0.018784815073013307,
      "rewards/classifier_reward": 0.9929000020027161,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 281
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 228.11429748535156,
      "epoch": 0.705,
      "grad_norm": 0.5186142582429601,
      "kl": 1.41875,
      "learning_rate": 2.3065434806862805e-07,
      "loss": 0.0038,
      "num_tokens": 7287768.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 282
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 228.11429443359376,
      "epoch": 0.7075,
      "grad_norm": 2.654334342756894,
      "kl": 1.0203125,
      "learning_rate": 2.2708734065768486e-07,
      "loss": 0.0029,
      "num_tokens": 7312659.0,
      "reward": 2.9993388175964357,
      "reward_std": 0.0017499331384897231,
      "rewards/classifier_reward": 0.9993385910987854,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 283
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 234.40001220703124,
      "epoch": 0.71,
      "grad_norm": 4.588949960274673,
      "kl": 1.12421875,
      "learning_rate": 2.2354000836581831e-07,
      "loss": 0.0021,
      "num_tokens": 7338617.0,
      "reward": 2.9408255100250242,
      "reward_std": 0.1565615115687251,
      "rewards/classifier_reward": 0.9908253073692321,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 284
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 231.6285827636719,
      "epoch": 0.7125,
      "grad_norm": 0.21793328593853228,
      "kl": 1.21484375,
      "learning_rate": 2.2001260693120232e-07,
      "loss": 0.0036,
      "num_tokens": 7364198.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 285
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 240.08572998046876,
      "epoch": 0.715,
      "grad_norm": 2.616590063333536,
      "kl": 1.01640625,
      "learning_rate": 2.1650539065513412e-07,
      "loss": 0.0029,
      "num_tokens": 7390479.0,
      "reward": 2.978571653366089,
      "reward_std": 0.056694668531417844,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 286
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 241.97144165039063,
      "epoch": 0.7175,
      "grad_norm": 4.531348675554103,
      "kl": 1.19375,
      "learning_rate": 2.1301861238370016e-07,
      "loss": 0.0031,
      "num_tokens": 7416732.0,
      "reward": 2.942857360839844,
      "reward_std": 0.09759000539779664,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9428571462631226,
      "rewards/slop_reward": 1.0,
      "step": 287
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 246.20001220703125,
      "epoch": 0.72,
      "grad_norm": 4.718898706696491,
      "kl": 1.49921875,
      "learning_rate": 2.0955252348954805e-07,
      "loss": 0.0034,
      "num_tokens": 7443268.0,
      "reward": 2.6439733505249023,
      "reward_std": 0.043535226583480836,
      "rewards/classifier_reward": 0.8439731419086456,
      "rewards/length_reward": 0.8,
      "rewards/slop_reward": 1.0,
      "step": 288
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 224.48572387695313,
      "epoch": 0.7225,
      "grad_norm": 0.04294906761504897,
      "kl": 1.00546875,
      "learning_rate": 2.0610737385376348e-07,
      "loss": 0.0034,
      "num_tokens": 7468990.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 289
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 235.77144470214844,
      "epoch": 0.725,
      "grad_norm": 0.13090449233770926,
      "kl": 1.19453125,
      "learning_rate": 2.026834118478567e-07,
      "loss": 0.0036,
      "num_tokens": 7495162.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 290
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 222.57143859863282,
      "epoch": 0.7275,
      "grad_norm": 0.04429936575325668,
      "kl": 0.97734375,
      "learning_rate": 1.9928088431585589e-07,
      "loss": 0.0034,
      "num_tokens": 7520868.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 291
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 238.7428741455078,
      "epoch": 0.73,
      "grad_norm": 12.126930606187928,
      "kl": 10.5515625,
      "learning_rate": 1.959000365565119e-07,
      "loss": 0.0129,
      "num_tokens": 7546905.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 292
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 231.20001220703125,
      "epoch": 0.7325,
      "grad_norm": 1.1227825049824671,
      "kl": 1.8796875,
      "learning_rate": 1.925411123056128e-07,
      "loss": 0.0043,
      "num_tokens": 7572334.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 293
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 234.68572387695312,
      "epoch": 0.735,
      "grad_norm": 2.907894335014808,
      "kl": 1.3328125,
      "learning_rate": 1.8920435371841392e-07,
      "loss": 0.0032,
      "num_tokens": 7598444.0,
      "reward": 2.996587371826172,
      "reward_std": 0.009029625356197358,
      "rewards/classifier_reward": 0.9965871214866638,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 294
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 232.68572387695312,
      "epoch": 0.7375,
      "grad_norm": 0.10333954670768886,
      "kl": 1.1203125,
      "learning_rate": 1.858900013521788e-07,
      "loss": 0.0035,
      "num_tokens": 7624449.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 295
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 233.54286499023436,
      "epoch": 0.74,
      "grad_norm": 0.058548069484880644,
      "kl": 1.05859375,
      "learning_rate": 1.8259829414883725e-07,
      "loss": 0.0034,
      "num_tokens": 7650523.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 296
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 235.9428680419922,
      "epoch": 0.7425,
      "grad_norm": 3.1888591778113993,
      "kl": 1.009375,
      "learning_rate": 1.7932946941775878e-07,
      "loss": 0.0029,
      "num_tokens": 7676533.0,
      "reward": 2.978571653366089,
      "reward_std": 0.056694668531417844,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 297
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 235.5428680419922,
      "epoch": 0.745,
      "grad_norm": 0.10062491596572344,
      "kl": 1.0046875,
      "learning_rate": 1.7608376281864502e-07,
      "loss": 0.0034,
      "num_tokens": 7702619.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 298
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 244.51430053710936,
      "epoch": 0.7475,
      "grad_norm": 0.061386216937859915,
      "kl": 1.02578125,
      "learning_rate": 1.7286140834453954e-07,
      "loss": 0.0034,
      "num_tokens": 7729097.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 299
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 254.4571533203125,
      "epoch": 0.75,
      "grad_norm": 0.05751560507113229,
      "kl": 1.03984375,
      "learning_rate": 1.6966263830495935e-07,
      "loss": 0.0034,
      "num_tokens": 7755641.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 300
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 240.34287109375,
      "epoch": 0.7525,
      "grad_norm": 0.11118968278237154,
      "kl": 1.11484375,
      "learning_rate": 1.6648768330914576e-07,
      "loss": 0.0035,
      "num_tokens": 7781895.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 301
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 241.40000915527344,
      "epoch": 0.755,
      "grad_norm": 0.191477055644317,
      "kl": 1.0296875,
      "learning_rate": 1.6333677224944037e-07,
      "loss": 0.0034,
      "num_tokens": 7808096.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 302
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 256.8571533203125,
      "epoch": 0.7575,
      "grad_norm": 2.9503904549206177,
      "kl": 0.984375,
      "learning_rate": 1.6021013228478275e-07,
      "loss": 0.0029,
      "num_tokens": 7835006.0,
      "reward": 2.9714287757873534,
      "reward_std": 0.07559289336204529,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 1.0,
      "step": 303
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 246.02857971191406,
      "epoch": 0.76,
      "grad_norm": 7.0345652170290975,
      "kl": 1.221875,
      "learning_rate": 1.5710798882433428e-07,
      "loss": 0.0036,
      "num_tokens": 7861536.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 304
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 253.68572387695312,
      "epoch": 0.7625,
      "grad_norm": 0.0476505587038927,
      "kl": 0.96328125,
      "learning_rate": 1.5403056551122694e-07,
      "loss": 0.0033,
      "num_tokens": 7888255.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 305
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 247.88572387695314,
      "epoch": 0.765,
      "grad_norm": 0.04204135010045963,
      "kl": 0.89609375,
      "learning_rate": 1.5097808420644115e-07,
      "loss": 0.0033,
      "num_tokens": 7914639.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 306
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 255.1428649902344,
      "epoch": 0.7675,
      "grad_norm": 0.07577358460759685,
      "kl": 0.9578125,
      "learning_rate": 1.479507649728105e-07,
      "loss": 0.0033,
      "num_tokens": 7941362.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 307
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 249.9714385986328,
      "epoch": 0.77,
      "grad_norm": 0.04375804621751843,
      "kl": 0.95703125,
      "learning_rate": 1.4494882605915714e-07,
      "loss": 0.0033,
      "num_tokens": 7967870.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 308
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 262.4571533203125,
      "epoch": 0.7725,
      "grad_norm": 3.881559830490846,
      "kl": 1.06953125,
      "learning_rate": 1.419724838845569e-07,
      "loss": 0.0025,
      "num_tokens": 7994976.0,
      "reward": 2.7243717670440675,
      "reward_std": 0.2547113478183746,
      "rewards/classifier_reward": 0.9529429793357849,
      "rewards/length_reward": 0.7714285761117935,
      "rewards/slop_reward": 1.0,
      "step": 309
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 256.0857208251953,
      "epoch": 0.775,
      "grad_norm": 0.042854049785843215,
      "kl": 0.934375,
      "learning_rate": 1.3902195302273778e-07,
      "loss": 0.0033,
      "num_tokens": 8021851.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 310
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 270.20001525878905,
      "epoch": 0.7775,
      "grad_norm": 5.120776391757703,
      "kl": 0.9703125,
      "learning_rate": 1.3609744618661013e-07,
      "loss": 0.0019,
      "num_tokens": 8049101.0,
      "reward": 2.8000001430511476,
      "reward_std": 0.28008740544319155,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.8000000059604645,
      "rewards/slop_reward": 1.0,
      "step": 311
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 266.88572998046874,
      "epoch": 0.78,
      "grad_norm": 5.825629703539792,
      "kl": 1.01640625,
      "learning_rate": 1.331991742129318e-07,
      "loss": 0.0024,
      "num_tokens": 8076201.0,
      "reward": 2.828571605682373,
      "reward_std": 0.1731828987598419,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.8285714328289032,
      "rewards/slop_reward": 1.0,
      "step": 312
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 255.80001220703124,
      "epoch": 0.7825,
      "grad_norm": 2.2750579624967204,
      "kl": 0.90625,
      "learning_rate": 1.3032734604710783e-07,
      "loss": 0.0028,
      "num_tokens": 8102845.0,
      "reward": 2.978571653366089,
      "reward_std": 0.056694668531417844,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 313
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 258.34287109375,
      "epoch": 0.785,
      "grad_norm": 2.9657377944921426,
      "kl": 1.23046875,
      "learning_rate": 1.2748216872812745e-07,
      "loss": 0.0031,
      "num_tokens": 8129806.0,
      "reward": 2.828571653366089,
      "reward_std": 0.07559289336204529,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.8285714298486709,
      "rewards/slop_reward": 1.0,
      "step": 314
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 241.60000915527343,
      "epoch": 0.7875,
      "grad_norm": 0.04358713190283898,
      "kl": 0.9265625,
      "learning_rate": 1.2466384737363779e-07,
      "loss": 0.0033,
      "num_tokens": 8156161.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 315
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 252.94286499023437,
      "epoch": 0.79,
      "grad_norm": 0.05992737661065536,
      "kl": 0.98046875,
      "learning_rate": 1.2187258516515642e-07,
      "loss": 0.0034,
      "num_tokens": 8182699.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 316
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 247.91429443359374,
      "epoch": 0.7925,
      "grad_norm": 0.07122711325744925,
      "kl": 0.9984375,
      "learning_rate": 1.1910858333342277e-07,
      "loss": 0.0034,
      "num_tokens": 8209296.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 317
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 250.11429748535156,
      "epoch": 0.795,
      "grad_norm": 3.4982043756301584,
      "kl": 0.98671875,
      "learning_rate": 1.1637204114389177e-07,
      "loss": 0.0029,
      "num_tokens": 8235818.0,
      "reward": 2.9714287757873534,
      "reward_std": 0.07559289336204529,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 1.0,
      "step": 318
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 251.08572387695312,
      "epoch": 0.7975,
      "grad_norm": 0.04511891765892185,
      "kl": 0.93671875,
      "learning_rate": 1.1366315588236741e-07,
      "loss": 0.0033,
      "num_tokens": 8262480.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 319
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 252.4571563720703,
      "epoch": 0.8,
      "grad_norm": 4.101920254520037,
      "kl": 0.98984375,
      "learning_rate": 1.1098212284078035e-07,
      "loss": 0.0024,
      "num_tokens": 8289236.0,
      "reward": 2.8857144832611086,
      "reward_std": 0.19518001079559327,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.8857142925262451,
      "rewards/slop_reward": 1.0,
      "step": 320
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 248.91429748535157,
      "epoch": 0.8025,
      "grad_norm": 0.05706184089424821,
      "kl": 0.975,
      "learning_rate": 1.0832913530310783e-07,
      "loss": 0.0034,
      "num_tokens": 8315716.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 321
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 250.31430358886718,
      "epoch": 0.805,
      "grad_norm": 0.07917877336969092,
      "kl": 1.02265625,
      "learning_rate": 1.0570438453144043e-07,
      "loss": 0.0034,
      "num_tokens": 8342093.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 322
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 246.60001220703126,
      "epoch": 0.8075,
      "grad_norm": 0.09776954891266137,
      "kl": 1.1421875,
      "learning_rate": 1.0310805975219255e-07,
      "loss": 0.0035,
      "num_tokens": 8368479.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 323
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 241.85715026855468,
      "epoch": 0.81,
      "grad_norm": 0.05012984603712916,
      "kl": 0.96015625,
      "learning_rate": 1.0054034814246093e-07,
      "loss": 0.0033,
      "num_tokens": 8394862.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 324
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 244.7714385986328,
      "epoch": 0.8125,
      "grad_norm": 0.2165819917911517,
      "kl": 1.35078125,
      "learning_rate": 9.800143481652979e-08,
      "loss": 0.0037,
      "num_tokens": 8421276.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 325
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 246.08572998046876,
      "epoch": 0.815,
      "grad_norm": 0.20309822089295643,
      "kl": 1.27265625,
      "learning_rate": 9.549150281252632e-08,
      "loss": 0.0037,
      "num_tokens": 8447767.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 326
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 243.6571533203125,
      "epoch": 0.8175,
      "grad_norm": 0.15578036977279336,
      "kl": 1.1515625,
      "learning_rate": 9.30107330792243e-08,
      "loss": 0.0035,
      "num_tokens": 8474150.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 327
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 237.4857208251953,
      "epoch": 0.82,
      "grad_norm": 0.043416576178703016,
      "kl": 0.9265625,
      "learning_rate": 9.055930446299914e-08,
      "loss": 0.0033,
      "num_tokens": 8500171.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 328
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 234.57143859863282,
      "epoch": 0.8225,
      "grad_norm": 0.05061080629519643,
      "kl": 1.034375,
      "learning_rate": 8.813739369493395e-08,
      "loss": 0.0034,
      "num_tokens": 8526178.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 329
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 243.28572692871094,
      "epoch": 0.825,
      "grad_norm": 0.04111158267935015,
      "kl": 0.94296875,
      "learning_rate": 8.574517537807896e-08,
      "loss": 0.0033,
      "num_tokens": 8552519.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 330
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 242.00000915527343,
      "epoch": 0.8275,
      "grad_norm": 0.3393008711951354,
      "kl": 1.38515625,
      "learning_rate": 8.338282197486362e-08,
      "loss": 0.0038,
      "num_tokens": 8578538.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 331
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 249.71430053710938,
      "epoch": 0.83,
      "grad_norm": 5.920098814946804,
      "kl": 1.471875,
      "learning_rate": 8.105050379466332e-08,
      "loss": 0.0034,
      "num_tokens": 8604935.0,
      "reward": 2.9714287757873534,
      "reward_std": 0.07559289336204529,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 1.0,
      "step": 332
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 249.34286499023438,
      "epoch": 0.8325,
      "grad_norm": 2.5733896209901923,
      "kl": 1.1671875,
      "learning_rate": 7.87483889815207e-08,
      "loss": 0.0031,
      "num_tokens": 8631504.0,
      "reward": 2.991787624359131,
      "reward_std": 0.02172858864068985,
      "rewards/classifier_reward": 0.9917873620986939,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 333
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 237.14287109375,
      "epoch": 0.835,
      "grad_norm": 2.715010668750814,
      "kl": 0.91875,
      "learning_rate": 7.64766435020246e-08,
      "loss": 0.0028,
      "num_tokens": 8657724.0,
      "reward": 2.989917850494385,
      "reward_std": 0.026675373315811157,
      "rewards/classifier_reward": 0.9899176597595215,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 334
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 247.05715637207032,
      "epoch": 0.8375,
      "grad_norm": 2.9706039903092303,
      "kl": 0.96171875,
      "learning_rate": 7.423543113334435e-08,
      "loss": 0.0029,
      "num_tokens": 8684291.0,
      "reward": 2.998081636428833,
      "reward_std": 0.005076154321432114,
      "rewards/classifier_reward": 0.9980813980102539,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 335
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 251.02857971191406,
      "epoch": 0.84,
      "grad_norm": 3.791827437250973,
      "kl": 0.978125,
      "learning_rate": 7.202491345142286e-08,
      "loss": 0.0029,
      "num_tokens": 8710997.0,
      "reward": 2.6675583362579345,
      "reward_std": 0.1484653353691101,
      "rewards/classifier_reward": 0.8389866888523102,
      "rewards/length_reward": 0.8285714298486709,
      "rewards/slop_reward": 1.0,
      "step": 336
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 241.48572692871093,
      "epoch": 0.8425,
      "grad_norm": 0.26642623759823414,
      "kl": 1.29609375,
      "learning_rate": 6.984524981932755e-08,
      "loss": 0.0037,
      "num_tokens": 8736817.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 337
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 247.2571533203125,
      "epoch": 0.845,
      "grad_norm": 0.05826256029366563,
      "kl": 0.9796875,
      "learning_rate": 6.769659737576227e-08,
      "loss": 0.0034,
      "num_tokens": 8763338.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 338
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 247.00001525878906,
      "epoch": 0.8475,
      "grad_norm": 0.0820068561247779,
      "kl": 0.91484375,
      "learning_rate": 6.557911102373809e-08,
      "loss": 0.0033,
      "num_tokens": 8789832.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 339
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 241.57143859863282,
      "epoch": 0.85,
      "grad_norm": 0.145851308869347,
      "kl": 1.04375,
      "learning_rate": 6.349294341940592e-08,
      "loss": 0.0034,
      "num_tokens": 8816056.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 340
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 249.6571533203125,
      "epoch": 0.8525,
      "grad_norm": 4.893160591979278,
      "kl": 1.1859375,
      "learning_rate": 6.143824496105121e-08,
      "loss": 0.0031,
      "num_tokens": 8842714.0,
      "reward": 2.9714287757873534,
      "reward_std": 0.07559289336204529,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 1.0,
      "step": 341
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 245.80001220703124,
      "epoch": 0.855,
      "grad_norm": 0.06508578960217941,
      "kl": 1.04375,
      "learning_rate": 5.941516377825101e-08,
      "loss": 0.0034,
      "num_tokens": 8869237.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 342
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 249.80000915527344,
      "epoch": 0.8575,
      "grad_norm": 4.184670329516298,
      "kl": 0.9828125,
      "learning_rate": 5.7423845721195184e-08,
      "loss": 0.0024,
      "num_tokens": 8895106.0,
      "reward": 2.950000190734863,
      "reward_std": 0.13228756189346313,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 343
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 247.22858276367188,
      "epoch": 0.86,
      "grad_norm": 0.04621548010483809,
      "kl": 0.96796875,
      "learning_rate": 5.546443435017145e-08,
      "loss": 0.0034,
      "num_tokens": 8921496.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 344
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 243.71429443359375,
      "epoch": 0.8625,
      "grad_norm": 3.0589783495407725,
      "kl": 0.93515625,
      "learning_rate": 5.353707092521581e-08,
      "loss": 0.0028,
      "num_tokens": 8947905.0,
      "reward": 2.994158411026001,
      "reward_std": 0.01545594185590744,
      "rewards/classifier_reward": 0.9941581964492798,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 345
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 244.94286499023437,
      "epoch": 0.865,
      "grad_norm": 5.554678417760662,
      "kl": 3.01484375,
      "learning_rate": 5.16418943959282e-08,
      "loss": 0.0054,
      "num_tokens": 8974370.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 346
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 244.7714416503906,
      "epoch": 0.8675,
      "grad_norm": 0.0645376640043755,
      "kl": 0.915625,
      "learning_rate": 4.9779041391455775e-08,
      "loss": 0.0033,
      "num_tokens": 9000853.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 347
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 245.8571563720703,
      "epoch": 0.87,
      "grad_norm": 0.1578429426456495,
      "kl": 1.0453125,
      "learning_rate": 4.794864621064265e-08,
      "loss": 0.0034,
      "num_tokens": 9027326.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 348
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 243.51429748535156,
      "epoch": 0.8725,
      "grad_norm": 0.09635841303768418,
      "kl": 1.05234375,
      "learning_rate": 4.615084081234799e-08,
      "loss": 0.0034,
      "num_tokens": 9053438.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 349
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 238.80000915527344,
      "epoch": 0.875,
      "grad_norm": 9.025341176830713,
      "kl": 0.93828125,
      "learning_rate": 4.4385754805932095e-08,
      "loss": 0.0024,
      "num_tokens": 9079245.0,
      "reward": 2.9694100856781005,
      "reward_std": 0.0809337928891182,
      "rewards/classifier_reward": 0.9908384680747986,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 350
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 249.28572692871094,
      "epoch": 0.8775,
      "grad_norm": 3.7703349335107053,
      "kl": 1.09296875,
      "learning_rate": 4.2653515441913646e-08,
      "loss": 0.003,
      "num_tokens": 9105890.0,
      "reward": 2.9714287757873534,
      "reward_std": 0.07559289336204529,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 1.0,
      "step": 351
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 229.60000915527343,
      "epoch": 0.88,
      "grad_norm": 4.44121260340152,
      "kl": 0.9359375,
      "learning_rate": 4.095424760279453e-08,
      "loss": 0.0024,
      "num_tokens": 9131259.0,
      "reward": 2.8642487049102785,
      "reward_std": 0.13740314245224,
      "rewards/classifier_reward": 0.8642484605312347,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 352
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 238.171435546875,
      "epoch": 0.8825,
      "grad_norm": 4.315109786774216,
      "kl": 1.1671875,
      "learning_rate": 3.928807379405763e-08,
      "loss": 0.0026,
      "num_tokens": 9157515.0,
      "reward": 2.804906415939331,
      "reward_std": 0.016342369094491004,
      "rewards/classifier_reward": 0.8049062207341194,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 353
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 239.2571563720703,
      "epoch": 0.885,
      "grad_norm": 2.5953487008425773,
      "kl": 0.98125,
      "learning_rate": 3.7655114135334284e-08,
      "loss": 0.0029,
      "num_tokens": 9183753.0,
      "reward": 2.9993186473846434,
      "reward_std": 0.0018032947555184364,
      "rewards/classifier_reward": 0.9993184208869934,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 354
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 243.91429748535157,
      "epoch": 0.8875,
      "grad_norm": 3.061039239899815,
      "kl": 0.87109375,
      "learning_rate": 3.6055486351745324e-08,
      "loss": 0.0028,
      "num_tokens": 9210003.0,
      "reward": 2.9983937740325928,
      "reward_std": 0.004250280186533928,
      "rewards/classifier_reward": 0.9983935475349426,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 355
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 248.65715637207032,
      "epoch": 0.89,
      "grad_norm": 0.07280216811639449,
      "kl": 1.0234375,
      "learning_rate": 3.448930576541309e-08,
      "loss": 0.0034,
      "num_tokens": 9236471.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 356
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 244.571435546875,
      "epoch": 0.8925,
      "grad_norm": 0.04031184293990925,
      "kl": 0.85625,
      "learning_rate": 3.295668528714801e-08,
      "loss": 0.0032,
      "num_tokens": 9262896.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 357
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 247.0571533203125,
      "epoch": 0.895,
      "grad_norm": 0.048980473886171605,
      "kl": 0.90390625,
      "learning_rate": 3.145773540830815e-08,
      "loss": 0.0033,
      "num_tokens": 9289145.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 358
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 247.48572692871093,
      "epoch": 0.8975,
      "grad_norm": 0.06968257994497579,
      "kl": 0.9921875,
      "learning_rate": 2.9992564192834246e-08,
      "loss": 0.0034,
      "num_tokens": 9315727.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 359
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 245.34286804199218,
      "epoch": 0.9,
      "grad_norm": 0.05048579158042856,
      "kl": 0.965625,
      "learning_rate": 2.8561277269457895e-08,
      "loss": 0.0034,
      "num_tokens": 9342091.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 360
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 240.94287109375,
      "epoch": 0.9025,
      "grad_norm": 0.07338683112545501,
      "kl": 0.9578125,
      "learning_rate": 2.7163977824087692e-08,
      "loss": 0.0033,
      "num_tokens": 9368444.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 361
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 245.48572387695313,
      "epoch": 0.905,
      "grad_norm": 2.0759195339474985,
      "kl": 2.634375,
      "learning_rate": 2.5800766592369073e-08,
      "loss": 0.005,
      "num_tokens": 9394956.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 362
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 243.1428649902344,
      "epoch": 0.9075,
      "grad_norm": 0.071979853158223,
      "kl": 0.9921875,
      "learning_rate": 2.4471741852423233e-08,
      "loss": 0.0034,
      "num_tokens": 9421111.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 363
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 247.60001525878906,
      "epoch": 0.91,
      "grad_norm": 4.112340756036376,
      "kl": 1.06640625,
      "learning_rate": 2.3176999417760633e-08,
      "loss": 0.003,
      "num_tokens": 9447147.0,
      "reward": 2.997753620147705,
      "reward_std": 0.005943871289491654,
      "rewards/classifier_reward": 0.9977534294128418,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 364
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 247.65715637207032,
      "epoch": 0.9125,
      "grad_norm": 0.1514075169088936,
      "kl": 1.1046875,
      "learning_rate": 2.1916632630374577e-08,
      "loss": 0.0035,
      "num_tokens": 9473085.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 365
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 240.74286804199218,
      "epoch": 0.915,
      "grad_norm": 0.18737733575482768,
      "kl": 1.07578125,
      "learning_rate": 2.0690732354011088e-08,
      "loss": 0.0035,
      "num_tokens": 9499431.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 366
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 235.6571533203125,
      "epoch": 0.9175,
      "grad_norm": 0.04304702113082608,
      "kl": 0.92421875,
      "learning_rate": 1.9499386967619104e-08,
      "loss": 0.0033,
      "num_tokens": 9525433.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 367
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 234.6571502685547,
      "epoch": 0.92,
      "grad_norm": 2.545151028403328,
      "kl": 0.9734375,
      "learning_rate": 1.8342682358978068e-08,
      "loss": 0.0029,
      "num_tokens": 9551354.0,
      "reward": 2.9982373237609865,
      "reward_std": 0.004664153978228569,
      "rewards/classifier_reward": 0.9982371211051941,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 368
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 244.11429443359376,
      "epoch": 0.9225,
      "grad_norm": 0.04345300022978855,
      "kl": 0.9125,
      "learning_rate": 1.7220701918506662e-08,
      "loss": 0.0033,
      "num_tokens": 9577818.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 369
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 247.74287109375,
      "epoch": 0.925,
      "grad_norm": 3.5704547668159674,
      "kl": 1.284375,
      "learning_rate": 1.6133526533250563e-08,
      "loss": 0.0032,
      "num_tokens": 9604325.0,
      "reward": 2.9714287757873534,
      "reward_std": 0.07559289336204529,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 0.9714285731315613,
      "rewards/slop_reward": 1.0,
      "step": 370
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 237.11429748535156,
      "epoch": 0.9275,
      "grad_norm": 0.041173535714356384,
      "kl": 0.88828125,
      "learning_rate": 1.5081234581051482e-08,
      "loss": 0.0033,
      "num_tokens": 9630168.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 371
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 244.8571533203125,
      "epoch": 0.93,
      "grad_norm": 0.08243253183651265,
      "kl": 0.934375,
      "learning_rate": 1.4063901924895982e-08,
      "loss": 0.0033,
      "num_tokens": 9656156.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 372
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 237.1428649902344,
      "epoch": 0.9325,
      "grad_norm": 0.05082406017565479,
      "kl": 0.94921875,
      "learning_rate": 1.3081601907447004e-08,
      "loss": 0.0033,
      "num_tokens": 9682368.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 373
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 242.80001220703124,
      "epoch": 0.935,
      "grad_norm": 2.8407093679191733,
      "kl": 1.04453125,
      "learning_rate": 1.2134405345755772e-08,
      "loss": 0.003,
      "num_tokens": 9708489.0,
      "reward": 2.978571653366089,
      "reward_std": 0.056694668531417844,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 374
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 248.08572692871093,
      "epoch": 0.9375,
      "grad_norm": 0.14511834175077776,
      "kl": 1.07421875,
      "learning_rate": 1.1222380526156927e-08,
      "loss": 0.0035,
      "num_tokens": 9734849.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 375
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 241.48572692871093,
      "epoch": 0.94,
      "grad_norm": 0.042968063146662946,
      "kl": 0.91875,
      "learning_rate": 1.034559319934497e-08,
      "loss": 0.0033,
      "num_tokens": 9761221.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 376
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 235.17144165039062,
      "epoch": 0.9425,
      "grad_norm": 3.168647847961566,
      "kl": 1.0890625,
      "learning_rate": 9.504106575634663e-09,
      "loss": 0.003,
      "num_tokens": 9787372.0,
      "reward": 2.965010404586792,
      "reward_std": 0.06012881994247436,
      "rewards/classifier_reward": 0.9864387512207031,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 377
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 237.91429748535157,
      "epoch": 0.945,
      "grad_norm": 0.04526250685823323,
      "kl": 0.99375,
      "learning_rate": 8.697981320403336e-09,
      "loss": 0.0034,
      "num_tokens": 9813619.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 378
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 246.20001220703125,
      "epoch": 0.9475,
      "grad_norm": 4.273461002843142,
      "kl": 0.9484375,
      "learning_rate": 7.927275549718226e-09,
      "loss": 0.0019,
      "num_tokens": 9839751.0,
      "reward": 2.9877804279327393,
      "reward_std": 0.03233038559556008,
      "rewards/classifier_reward": 0.9877802491188049,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 379
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 247.8571533203125,
      "epoch": 0.95,
      "grad_norm": 0.20002216578085852,
      "kl": 1.165625,
      "learning_rate": 7.1920448261457715e-09,
      "loss": 0.0035,
      "num_tokens": 9866244.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 380
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 244.60000915527343,
      "epoch": 0.9525,
      "grad_norm": 0.0834845425154776,
      "kl": 1.04375,
      "learning_rate": 6.492342154746588e-09,
      "loss": 0.0034,
      "num_tokens": 9892505.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 381
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 237.31429748535157,
      "epoch": 0.955,
      "grad_norm": 2.52753839477545,
      "kl": 1.11171875,
      "learning_rate": 5.828217979253869e-09,
      "loss": 0.003,
      "num_tokens": 9918307.0,
      "reward": 2.9943798542022706,
      "reward_std": 0.014869998395442962,
      "rewards/classifier_reward": 0.9943796753883362,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 382
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 235.571435546875,
      "epoch": 0.9575,
      "grad_norm": 3.128240693093393,
      "kl": 0.98359375,
      "learning_rate": 5.1997201784368395e-09,
      "loss": 0.0029,
      "num_tokens": 9944374.0,
      "reward": 2.995487594604492,
      "reward_std": 0.011939284205436707,
      "rewards/classifier_reward": 0.995487380027771,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 383
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 240.9428680419922,
      "epoch": 0.96,
      "grad_norm": 3.5538024950839295,
      "kl": 1.0,
      "learning_rate": 4.606894062648969e-09,
      "loss": 0.0024,
      "num_tokens": 9970727.0,
      "reward": 2.977288818359375,
      "reward_std": 0.06008877456188202,
      "rewards/classifier_reward": 0.9987171411514282,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 384
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 240.6571502685547,
      "epoch": 0.9625,
      "grad_norm": 0.1991534915494115,
      "kl": 1.184375,
      "learning_rate": 4.049782370561583e-09,
      "loss": 0.0036,
      "num_tokens": 9997070.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 385
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 235.34286804199218,
      "epoch": 0.965,
      "grad_norm": 0.05753950305238813,
      "kl": 0.99375,
      "learning_rate": 3.5284252660823244e-09,
      "loss": 0.0034,
      "num_tokens": 10023227.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 386
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 237.6571533203125,
      "epoch": 0.9675,
      "grad_norm": 0.09059748000607612,
      "kl": 1.003125,
      "learning_rate": 3.0428603354600844e-09,
      "loss": 0.0034,
      "num_tokens": 10049465.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 387
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 254.54286499023436,
      "epoch": 0.97,
      "grad_norm": 0.12475756025548783,
      "kl": 1.07890625,
      "learning_rate": 2.5931225845748917e-09,
      "loss": 0.0035,
      "num_tokens": 10076294.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 388
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 242.6571533203125,
      "epoch": 0.9725,
      "grad_norm": 0.4783333308376495,
      "kl": 1.3875,
      "learning_rate": 2.1792444364144847e-09,
      "loss": 0.0038,
      "num_tokens": 10102419.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 389
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 241.34286804199218,
      "epoch": 0.975,
      "grad_norm": 0.08716408529439204,
      "kl": 0.93125,
      "learning_rate": 1.8012557287367391e-09,
      "loss": 0.0033,
      "num_tokens": 10128786.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 390
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 241.00000915527343,
      "epoch": 0.9775,
      "grad_norm": 3.1645439252715826,
      "kl": 0.9140625,
      "learning_rate": 1.4591837119186102e-09,
      "loss": 0.0028,
      "num_tokens": 10155141.0,
      "reward": 2.978571653366089,
      "reward_std": 0.056694668531417844,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 391
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 235.28572692871094,
      "epoch": 0.98,
      "grad_norm": 0.41518239333595675,
      "kl": 1.1234375,
      "learning_rate": 1.1530530469914256e-09,
      "loss": 0.0035,
      "num_tokens": 10181268.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 392
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 243.7714385986328,
      "epoch": 0.9825,
      "grad_norm": 0.040861855208901524,
      "kl": 0.878125,
      "learning_rate": 8.828858038632536e-10,
      "loss": 0.0033,
      "num_tokens": 10207349.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 393
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 249.9714385986328,
      "epoch": 0.985,
      "grad_norm": 0.12170276230850544,
      "kl": 1.1421875,
      "learning_rate": 6.48701459727563e-10,
      "loss": 0.0035,
      "num_tokens": 10233544.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 394
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 243.00001220703126,
      "epoch": 0.9875,
      "grad_norm": 0.04242721259201106,
      "kl": 0.93828125,
      "learning_rate": 4.5051689765929213e-10,
      "loss": 0.0033,
      "num_tokens": 10259969.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 395
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 248.68572387695312,
      "epoch": 0.99,
      "grad_norm": 2.430119389458536,
      "kl": 0.96875,
      "learning_rate": 2.883464053973772e-10,
      "loss": 0.0029,
      "num_tokens": 10286476.0,
      "reward": 2.9996359825134276,
      "reward_std": 0.0009637950919568538,
      "rewards/classifier_reward": 0.9996357202529907,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 396
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 244.34286499023438,
      "epoch": 0.9925,
      "grad_norm": 2.4946704769049948,
      "kl": 1.090625,
      "learning_rate": 1.6220167431502118e-10,
      "loss": 0.003,
      "num_tokens": 10312797.0,
      "reward": 2.998571014404297,
      "reward_std": 0.0037812769412994387,
      "rewards/classifier_reward": 0.9985708117485046,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 397
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 240.60001220703126,
      "epoch": 0.995,
      "grad_norm": 0.057929252615434516,
      "kl": 0.93359375,
      "learning_rate": 7.209179857675663e-11,
      "loss": 0.0033,
      "num_tokens": 10338970.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 398
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 238.6285827636719,
      "epoch": 0.9975,
      "grad_norm": 0.06529828900649211,
      "kl": 1.0125,
      "learning_rate": 1.8023274482636965e-11,
      "loss": 0.0034,
      "num_tokens": 10365242.0,
      "reward": 3.000000238418579,
      "reward_std": 0.0,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 1.0,
      "step": 399
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 239.8800018310547,
      "epoch": 1.0,
      "grad_norm": 2.484762868251253,
      "kl": 0.875,
      "learning_rate": 0.0,
      "loss": 0.0028,
      "num_tokens": 10384026.0,
      "reward": 2.978571653366089,
      "reward_std": 0.056694668531417844,
      "rewards/classifier_reward": 1.0,
      "rewards/length_reward": 1.0,
      "rewards/slop_reward": 0.9785714268684387,
      "step": 400
    }
  ],
  "logging_steps": 1,
  "max_steps": 400,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}