| { | |
| "best_global_step": null, | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.0, | |
| "eval_steps": 500, | |
| "global_step": 500, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "clip_ratio": 0.003959985822439194, | |
| "completion_length": 54.13888931274414, | |
| "epoch": 0.01, | |
| "grad_norm": 784750.1992774517, | |
| "kl": 785907.2, | |
| "learning_rate": 4e-07, | |
| "loss": 1571.24, | |
| "reward": 0.09333334118127823, | |
| "reward_std": 0.15150094280640283, | |
| "rewards/judge_tool_use": 0.09236111243565877, | |
| "rewards/judge_tool_use/std": 0.22503272195657095, | |
| "rewards/verify_correctness": 0.09722222325702508, | |
| "rewards/verify_correctness/std": 0.27655620872974396, | |
| "step": 5 | |
| }, | |
| { | |
| "clip_ratio": 0.007778534106910229, | |
| "completion_length": 70.86458587646484, | |
| "epoch": 0.02, | |
| "grad_norm": 7994810.896724847, | |
| "kl": 36229.2, | |
| "learning_rate": 9e-07, | |
| "loss": 72.2516, | |
| "reward": 0.1066666692495346, | |
| "reward_std": 0.19039028137922287, | |
| "rewards/judge_tool_use": 0.12291667237877846, | |
| "rewards/judge_tool_use/std": 0.2716349810361862, | |
| "rewards/verify_correctness": 0.0416666679084301, | |
| "rewards/verify_correctness/std": 0.20194092392921448, | |
| "step": 10 | |
| }, | |
| { | |
| "clip_ratio": 0.006301742233335972, | |
| "completion_length": 75.5069465637207, | |
| "epoch": 0.03, | |
| "grad_norm": 10255.043920889624, | |
| "kl": 26379.2, | |
| "learning_rate": 1e-06, | |
| "loss": 52.6755, | |
| "reward": 0.06888888776302338, | |
| "reward_std": 0.12580444912115732, | |
| "rewards/judge_tool_use": 0.07222222536802292, | |
| "rewards/judge_tool_use/std": 0.19341551760832468, | |
| "rewards/verify_correctness": 0.055555555348594986, | |
| "rewards/verify_correctness/std": 0.21832925577958426, | |
| "step": 15 | |
| }, | |
| { | |
| "clip_ratio": 0.005547903012484312, | |
| "completion_length": 63.864585876464844, | |
| "epoch": 0.04, | |
| "grad_norm": 12547.287212363904, | |
| "kl": 438.30462646484375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.8478, | |
| "reward": 0.03999999910593033, | |
| "reward_std": 0.06966040655970573, | |
| "rewards/judge_tool_use": 0.03437500377185643, | |
| "rewards/judge_tool_use/std": 0.10251733846962452, | |
| "rewards/verify_correctness": 0.0625, | |
| "rewards/verify_correctness/std": 0.16710934042930603, | |
| "step": 20 | |
| }, | |
| { | |
| "clip_ratio": 0.004662872105836868, | |
| "completion_length": 89.24305725097656, | |
| "epoch": 0.05, | |
| "grad_norm": 298314332.7497967, | |
| "kl": 204444.275, | |
| "learning_rate": 1e-06, | |
| "loss": 406.5101, | |
| "reward": 0.12250000486771266, | |
| "reward_std": 0.172040027876695, | |
| "rewards/judge_tool_use": 0.13055556019147238, | |
| "rewards/judge_tool_use/std": 0.2499552220106125, | |
| "rewards/verify_correctness": 0.09027777860562007, | |
| "rewards/verify_correctness/std": 0.22458433111508688, | |
| "step": 25 | |
| }, | |
| { | |
| "clip_ratio": 0.007842596806585788, | |
| "completion_length": 125.81250381469727, | |
| "epoch": 0.06, | |
| "grad_norm": 2920.4418807388934, | |
| "kl": 21399.8, | |
| "learning_rate": 1e-06, | |
| "loss": 42.6862, | |
| "reward": 0.16333334147930145, | |
| "reward_std": 0.17722879350185394, | |
| "rewards/judge_tool_use": 0.17812500149011612, | |
| "rewards/judge_tool_use/std": 0.2530943974852562, | |
| "rewards/verify_correctness": 0.1041666641831398, | |
| "rewards/verify_correctness/std": 0.20520704984664917, | |
| "step": 30 | |
| }, | |
| { | |
| "clip_ratio": 0.006266751885414123, | |
| "completion_length": 117.37500254313152, | |
| "epoch": 0.07, | |
| "grad_norm": 24639.61780550995, | |
| "kl": 91.1203125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.1544, | |
| "reward": 0.21361111104488373, | |
| "reward_std": 0.2253884275754293, | |
| "rewards/judge_tool_use": 0.23055555919806162, | |
| "rewards/judge_tool_use/std": 0.33459242184956867, | |
| "rewards/verify_correctness": 0.14583333830038706, | |
| "rewards/verify_correctness/std": 0.3359044094880422, | |
| "step": 35 | |
| }, | |
| { | |
| "clip_ratio": 0.008563124015927315, | |
| "completion_length": 122.10416793823242, | |
| "epoch": 0.08, | |
| "grad_norm": 220.2399474774772, | |
| "kl": 250.11875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.552, | |
| "reward": 0.21416668593883514, | |
| "reward_std": 0.21450992673635483, | |
| "rewards/judge_tool_use": 0.23125002533197403, | |
| "rewards/judge_tool_use/std": 0.30767104029655457, | |
| "rewards/verify_correctness": 0.14583333861082792, | |
| "rewards/verify_correctness/std": 0.2967151999473572, | |
| "step": 40 | |
| }, | |
| { | |
| "clip_ratio": 0.005935636721551419, | |
| "completion_length": 130.95833841959634, | |
| "epoch": 0.09, | |
| "grad_norm": 5460.174774182503, | |
| "kl": 244.15, | |
| "learning_rate": 1e-06, | |
| "loss": 0.5043, | |
| "reward": 0.2500000099341075, | |
| "reward_std": 0.23432709276676178, | |
| "rewards/judge_tool_use": 0.2743055472771327, | |
| "rewards/judge_tool_use/std": 0.3266789714495341, | |
| "rewards/verify_correctness": 0.1527777761220932, | |
| "rewards/verify_correctness/std": 0.36177847782770794, | |
| "step": 45 | |
| }, | |
| { | |
| "clip_ratio": 0.006385140307247639, | |
| "completion_length": 151.70833587646484, | |
| "epoch": 0.1, | |
| "grad_norm": 30.233107092288996, | |
| "kl": 5.925, | |
| "learning_rate": 1e-06, | |
| "loss": 0.187, | |
| "reward": 0.16625000536441803, | |
| "reward_std": 0.15779273211956024, | |
| "rewards/judge_tool_use": 0.17916666343808174, | |
| "rewards/judge_tool_use/std": 0.24784564971923828, | |
| "rewards/verify_correctness": 0.11458333395421505, | |
| "rewards/verify_correctness/std": 0.2981918603181839, | |
| "step": 50 | |
| }, | |
| { | |
| "clip_ratio": 0.005314220301806926, | |
| "completion_length": 133.38194529215494, | |
| "epoch": 0.11, | |
| "grad_norm": 116.04184304342279, | |
| "kl": 5.9447265625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0148, | |
| "reward": 0.27722224096457165, | |
| "reward_std": 0.2598785211642583, | |
| "rewards/judge_tool_use": 0.2979166582226753, | |
| "rewards/judge_tool_use/std": 0.3200907011826833, | |
| "rewards/verify_correctness": 0.1944444477558136, | |
| "rewards/verify_correctness/std": 0.37710362672805786, | |
| "step": 55 | |
| }, | |
| { | |
| "clip_ratio": 0.007683201320469379, | |
| "completion_length": 126.04166793823242, | |
| "epoch": 0.12, | |
| "grad_norm": 300.13848407753744, | |
| "kl": 2.870703125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0265, | |
| "reward": 0.22583334147930145, | |
| "reward_std": 0.18492110818624496, | |
| "rewards/judge_tool_use": 0.2562500014901161, | |
| "rewards/judge_tool_use/std": 0.3037812113761902, | |
| "rewards/verify_correctness": 0.1041666641831398, | |
| "rewards/verify_correctness/std": 0.300648495554924, | |
| "step": 60 | |
| }, | |
| { | |
| "clip_ratio": 0.0056252704933285715, | |
| "completion_length": 121.22916666666667, | |
| "epoch": 0.13, | |
| "grad_norm": 10523.81268433656, | |
| "kl": 35.8203125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0502, | |
| "reward": 0.30444446206092834, | |
| "reward_std": 0.20237582921981812, | |
| "rewards/judge_tool_use": 0.33541667461395264, | |
| "rewards/judge_tool_use/std": 0.3482949535051982, | |
| "rewards/verify_correctness": 0.18055555721124014, | |
| "rewards/verify_correctness/std": 0.3799951672554016, | |
| "step": 65 | |
| }, | |
| { | |
| "clip_ratio": 0.008523338288068772, | |
| "completion_length": 109.78125381469727, | |
| "epoch": 0.14, | |
| "grad_norm": 306.75431905722485, | |
| "kl": 54.10625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.1497, | |
| "reward": 0.37958335876464844, | |
| "reward_std": 0.27913033962249756, | |
| "rewards/judge_tool_use": 0.39374998211860657, | |
| "rewards/judge_tool_use/std": 0.3710695207118988, | |
| "rewards/verify_correctness": 0.3229166716337204, | |
| "rewards/verify_correctness/std": 0.47240525484085083, | |
| "step": 70 | |
| }, | |
| { | |
| "clip_ratio": 0.006300021335482598, | |
| "completion_length": 113.02777862548828, | |
| "epoch": 0.15, | |
| "grad_norm": 276.105755604053, | |
| "kl": 120.347265625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.2603, | |
| "reward": 0.3752777973810832, | |
| "reward_std": 0.23248672982056937, | |
| "rewards/judge_tool_use": 0.39097222685813904, | |
| "rewards/judge_tool_use/std": 0.39233259359995526, | |
| "rewards/verify_correctness": 0.3125, | |
| "rewards/verify_correctness/std": 0.4657697280248006, | |
| "step": 75 | |
| }, | |
| { | |
| "clip_ratio": 0.008843735605478287, | |
| "completion_length": 138.8854217529297, | |
| "epoch": 0.16, | |
| "grad_norm": 1275.1249001898307, | |
| "kl": 7.355078125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0298, | |
| "reward": 0.28958334028720856, | |
| "reward_std": 0.21345646679401398, | |
| "rewards/judge_tool_use": 0.3072916716337204, | |
| "rewards/judge_tool_use/std": 0.3630993217229843, | |
| "rewards/verify_correctness": 0.21875, | |
| "rewards/verify_correctness/std": 0.41601888835430145, | |
| "step": 80 | |
| }, | |
| { | |
| "clip_ratio": 0.007743672281503677, | |
| "completion_length": 101.95833587646484, | |
| "epoch": 0.17, | |
| "grad_norm": 890.3990436610322, | |
| "kl": 63141.15, | |
| "learning_rate": 1e-06, | |
| "loss": 125.8673, | |
| "reward": 0.5141666730244955, | |
| "reward_std": 0.22655925651391348, | |
| "rewards/judge_tool_use": 0.5298611124356588, | |
| "rewards/judge_tool_use/std": 0.38938658436139423, | |
| "rewards/verify_correctness": 0.4513888955116272, | |
| "rewards/verify_correctness/std": 0.4737345178922017, | |
| "step": 85 | |
| }, | |
| { | |
| "clip_ratio": 0.009472730197012424, | |
| "completion_length": 124.94791793823242, | |
| "epoch": 0.18, | |
| "grad_norm": 172.8478197001071, | |
| "kl": 7.838671875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0279, | |
| "reward": 0.49916666746139526, | |
| "reward_std": 0.17976997047662735, | |
| "rewards/judge_tool_use": 0.5250000059604645, | |
| "rewards/judge_tool_use/std": 0.3990606963634491, | |
| "rewards/verify_correctness": 0.3958333283662796, | |
| "rewards/verify_correctness/std": 0.4937349408864975, | |
| "step": 90 | |
| }, | |
| { | |
| "clip_ratio": 0.0054151594638824465, | |
| "completion_length": 127.84027862548828, | |
| "epoch": 0.19, | |
| "grad_norm": 2053.861574982554, | |
| "kl": 26665.05, | |
| "learning_rate": 1e-06, | |
| "loss": 53.3674, | |
| "reward": 0.3519444515307744, | |
| "reward_std": 0.16984381278355917, | |
| "rewards/judge_tool_use": 0.3791666825612386, | |
| "rewards/judge_tool_use/std": 0.35421761870384216, | |
| "rewards/verify_correctness": 0.24305555721124014, | |
| "rewards/verify_correctness/std": 0.4247280756632487, | |
| "step": 95 | |
| }, | |
| { | |
| "clip_ratio": 0.0074031388387084006, | |
| "completion_length": 107.06250381469727, | |
| "epoch": 0.2, | |
| "grad_norm": 79.32245833730725, | |
| "kl": 20.08984375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.049, | |
| "reward": 0.39124999940395355, | |
| "reward_std": 0.16201016306877136, | |
| "rewards/judge_tool_use": 0.41874998807907104, | |
| "rewards/judge_tool_use/std": 0.36362142860889435, | |
| "rewards/verify_correctness": 0.28125, | |
| "rewards/verify_correctness/std": 0.45300620794296265, | |
| "step": 100 | |
| }, | |
| { | |
| "clip_ratio": 0.00740175861865282, | |
| "completion_length": 97.12500254313152, | |
| "epoch": 0.21, | |
| "grad_norm": 31870.333648680495, | |
| "kl": 106.084375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.297, | |
| "reward": 0.38583337267239887, | |
| "reward_std": 0.20607438186804453, | |
| "rewards/judge_tool_use": 0.42152780294418335, | |
| "rewards/judge_tool_use/std": 0.36684316396713257, | |
| "rewards/verify_correctness": 0.2430555522441864, | |
| "rewards/verify_correctness/std": 0.41877804199854535, | |
| "step": 105 | |
| }, | |
| { | |
| "clip_ratio": 0.006064230762422085, | |
| "completion_length": 122.32292175292969, | |
| "epoch": 0.22, | |
| "grad_norm": 12.413314903140305, | |
| "kl": 28.44140625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0699, | |
| "reward": 0.33916667103767395, | |
| "reward_std": 0.11534206941723824, | |
| "rewards/judge_tool_use": 0.36666668951511383, | |
| "rewards/judge_tool_use/std": 0.3902362436056137, | |
| "rewards/verify_correctness": 0.2291666641831398, | |
| "rewards/verify_correctness/std": 0.41798071563243866, | |
| "step": 110 | |
| }, | |
| { | |
| "clip_ratio": 0.0057834615930914875, | |
| "completion_length": 118.7013931274414, | |
| "epoch": 0.23, | |
| "grad_norm": 10.45791929222894, | |
| "kl": 1.521484375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.025, | |
| "reward": 0.3566666742165883, | |
| "reward_std": 0.17904378970464072, | |
| "rewards/judge_tool_use": 0.39027777314186096, | |
| "rewards/judge_tool_use/std": 0.3494054178396861, | |
| "rewards/verify_correctness": 0.2222222238779068, | |
| "rewards/verify_correctness/std": 0.39503220717112225, | |
| "step": 115 | |
| }, | |
| { | |
| "clip_ratio": 0.008998725935816765, | |
| "completion_length": 109.9375, | |
| "epoch": 0.24, | |
| "grad_norm": 3.4155506219796594, | |
| "kl": 0.49375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0181, | |
| "reward": 0.4558333456516266, | |
| "reward_std": 0.1414913684129715, | |
| "rewards/judge_tool_use": 0.5072916746139526, | |
| "rewards/judge_tool_use/std": 0.38790474832057953, | |
| "rewards/verify_correctness": 0.2500000074505806, | |
| "rewards/verify_correctness/std": 0.43691860139369965, | |
| "step": 120 | |
| }, | |
| { | |
| "clip_ratio": 0.004882402718067169, | |
| "completion_length": 88.28472391764323, | |
| "epoch": 0.25, | |
| "grad_norm": 6.549383441404032, | |
| "kl": 0.5650390625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0137, | |
| "reward": 0.5822222232818604, | |
| "reward_std": 0.2408840556939443, | |
| "rewards/judge_tool_use": 0.6062499980131785, | |
| "rewards/judge_tool_use/std": 0.3277636071046193, | |
| "rewards/verify_correctness": 0.4861111094554265, | |
| "rewards/verify_correctness/std": 0.39317835370699566, | |
| "step": 125 | |
| }, | |
| { | |
| "clip_ratio": 0.007465552911162377, | |
| "completion_length": 111.97916793823242, | |
| "epoch": 0.26, | |
| "grad_norm": 89.72521188925481, | |
| "kl": 1.384375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0015, | |
| "reward": 0.476250022649765, | |
| "reward_std": 0.17444811761379242, | |
| "rewards/judge_tool_use": 0.5197916775941849, | |
| "rewards/judge_tool_use/std": 0.3664846569299698, | |
| "rewards/verify_correctness": 0.3020833358168602, | |
| "rewards/verify_correctness/std": 0.43010377883911133, | |
| "step": 130 | |
| }, | |
| { | |
| "clip_ratio": 0.006399596482515335, | |
| "completion_length": 96.55555979410808, | |
| "epoch": 0.27, | |
| "grad_norm": 1.7399274702030754, | |
| "kl": 0.966015625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0147, | |
| "reward": 0.4886111219724019, | |
| "reward_std": 0.23944897452990213, | |
| "rewards/judge_tool_use": 0.5430555542310079, | |
| "rewards/judge_tool_use/std": 0.3470989763736725, | |
| "rewards/verify_correctness": 0.2708333420256774, | |
| "rewards/verify_correctness/std": 0.37673963109652203, | |
| "step": 135 | |
| }, | |
| { | |
| "clip_ratio": 0.007890220731496811, | |
| "completion_length": 113.35416793823242, | |
| "epoch": 0.28, | |
| "grad_norm": 2.231161419407326, | |
| "kl": 3.628125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0151, | |
| "reward": 0.38749998807907104, | |
| "reward_std": 0.14481262117624283, | |
| "rewards/judge_tool_use": 0.484375, | |
| "rewards/judge_tool_use/std": 0.35279956459999084, | |
| "rewards/verify_correctness": 0.0, | |
| "rewards/verify_correctness/std": 0.0, | |
| "step": 140 | |
| }, | |
| { | |
| "clip_ratio": 0.004287280049175024, | |
| "completion_length": 128.59722391764322, | |
| "epoch": 0.29, | |
| "grad_norm": 5.750983107539032, | |
| "kl": 0.58671875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0151, | |
| "reward": 0.42972222963968915, | |
| "reward_std": 0.15125738829374313, | |
| "rewards/judge_tool_use": 0.47291667262713116, | |
| "rewards/judge_tool_use/std": 0.35496102770169574, | |
| "rewards/verify_correctness": 0.2569444477558136, | |
| "rewards/verify_correctness/std": 0.3278166850407918, | |
| "step": 145 | |
| }, | |
| { | |
| "clip_ratio": 0.007053372077643871, | |
| "completion_length": 98.94791793823242, | |
| "epoch": 0.3, | |
| "grad_norm": 1.5954937607260369, | |
| "kl": 0.237109375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0047, | |
| "reward": 0.5891667008399963, | |
| "reward_std": 0.14705681800842285, | |
| "rewards/judge_tool_use": 0.6322916746139526, | |
| "rewards/judge_tool_use/std": 0.37483371794223785, | |
| "rewards/verify_correctness": 0.4166666567325592, | |
| "rewards/verify_correctness/std": 0.49408674240112305, | |
| "step": 150 | |
| }, | |
| { | |
| "clip_ratio": 0.004191365744918585, | |
| "completion_length": 88.18055979410808, | |
| "epoch": 0.31, | |
| "grad_norm": 2.1141011322952803, | |
| "kl": 0.2513671875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.002, | |
| "reward": 0.7086111108462015, | |
| "reward_std": 0.14623582363128662, | |
| "rewards/judge_tool_use": 0.7486110925674438, | |
| "rewards/judge_tool_use/std": 0.3543677230676015, | |
| "rewards/verify_correctness": 0.5486110945542654, | |
| "rewards/verify_correctness/std": 0.4976752698421478, | |
| "step": 155 | |
| }, | |
| { | |
| "clip_ratio": 0.007304594852030277, | |
| "completion_length": 97.51041793823242, | |
| "epoch": 0.32, | |
| "grad_norm": 1.1578413021097405, | |
| "kl": 0.332421875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0083, | |
| "reward": 0.534166693687439, | |
| "reward_std": 0.11162800714373589, | |
| "rewards/judge_tool_use": 0.5843749940395355, | |
| "rewards/judge_tool_use/std": 0.3592209666967392, | |
| "rewards/verify_correctness": 0.3333333432674408, | |
| "rewards/verify_correctness/std": 0.47639307379722595, | |
| "step": 160 | |
| }, | |
| { | |
| "clip_ratio": 0.0060629287734627725, | |
| "completion_length": 110.75000254313152, | |
| "epoch": 0.33, | |
| "grad_norm": 0.8714902465835312, | |
| "kl": 0.35859375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0151, | |
| "reward": 0.5477778116861979, | |
| "reward_std": 0.13536312182744345, | |
| "rewards/judge_tool_use": 0.6013888915379842, | |
| "rewards/judge_tool_use/std": 0.38596779108047485, | |
| "rewards/verify_correctness": 0.3333333383003871, | |
| "rewards/verify_correctness/std": 0.45276867349942523, | |
| "step": 165 | |
| }, | |
| { | |
| "clip_ratio": 0.005478941276669503, | |
| "completion_length": 147.46875762939453, | |
| "epoch": 0.34, | |
| "grad_norm": 2.905066362863796, | |
| "kl": 0.1998046875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0213, | |
| "reward": 0.42500002682209015, | |
| "reward_std": 0.17469991743564606, | |
| "rewards/judge_tool_use": 0.4947916567325592, | |
| "rewards/judge_tool_use/std": 0.346432626247406, | |
| "rewards/verify_correctness": 0.1458333358168602, | |
| "rewards/verify_correctness/std": 0.35542023181915283, | |
| "step": 170 | |
| }, | |
| { | |
| "clip_ratio": 0.003099467512220144, | |
| "completion_length": 99.73611704508464, | |
| "epoch": 0.35, | |
| "grad_norm": 2.191551166799849, | |
| "kl": 0.3734375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0089, | |
| "reward": 0.7052778005599976, | |
| "reward_std": 0.11197900461653869, | |
| "rewards/judge_tool_use": 0.7652777632077535, | |
| "rewards/judge_tool_use/std": 0.26592855403820675, | |
| "rewards/verify_correctness": 0.4652777661879857, | |
| "rewards/verify_correctness/std": 0.4105065166950226, | |
| "step": 175 | |
| }, | |
| { | |
| "clip_ratio": 0.005960878264158964, | |
| "completion_length": 89.08333587646484, | |
| "epoch": 0.36, | |
| "grad_norm": 1.627066760583541, | |
| "kl": 0.53984375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0001, | |
| "reward": 0.6620833575725555, | |
| "reward_std": 0.15964354574680328, | |
| "rewards/judge_tool_use": 0.7312500178813934, | |
| "rewards/judge_tool_use/std": 0.31979653239250183, | |
| "rewards/verify_correctness": 0.3854166567325592, | |
| "rewards/verify_correctness/std": 0.49077247083187103, | |
| "step": 180 | |
| }, | |
| { | |
| "clip_ratio": 0.004781427606940269, | |
| "completion_length": 92.86805725097656, | |
| "epoch": 0.37, | |
| "grad_norm": 1.3401741313453097, | |
| "kl": 0.434765625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0005, | |
| "reward": 0.6336111426353455, | |
| "reward_std": 0.10186164081096649, | |
| "rewards/judge_tool_use": 0.6687500178813934, | |
| "rewards/judge_tool_use/std": 0.3347410162289937, | |
| "rewards/verify_correctness": 0.4930555423100789, | |
| "rewards/verify_correctness/std": 0.4855302969614665, | |
| "step": 185 | |
| }, | |
| { | |
| "clip_ratio": 0.006658449023962021, | |
| "completion_length": 108.47917175292969, | |
| "epoch": 0.38, | |
| "grad_norm": 2.5096982541940998, | |
| "kl": 0.5453125, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0017, | |
| "reward": 0.5404166877269745, | |
| "reward_std": 0.13422710448503494, | |
| "rewards/judge_tool_use": 0.5895833522081375, | |
| "rewards/judge_tool_use/std": 0.34020277857780457, | |
| "rewards/verify_correctness": 0.34375, | |
| "rewards/verify_correctness/std": 0.44986698031425476, | |
| "step": 190 | |
| }, | |
| { | |
| "clip_ratio": 0.004200516827404499, | |
| "completion_length": 95.00694783528645, | |
| "epoch": 0.39, | |
| "grad_norm": 1.0766041890185425, | |
| "kl": 0.31015625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0152, | |
| "reward": 0.6466667056083679, | |
| "reward_std": 0.18731083969275156, | |
| "rewards/judge_tool_use": 0.6972222328186035, | |
| "rewards/judge_tool_use/std": 0.3268563648064931, | |
| "rewards/verify_correctness": 0.4444444477558136, | |
| "rewards/verify_correctness/std": 0.4707372883955638, | |
| "step": 195 | |
| }, | |
| { | |
| "clip_ratio": 0.005295699182897806, | |
| "completion_length": 122.48958587646484, | |
| "epoch": 0.4, | |
| "grad_norm": 2.4546002318741573, | |
| "kl": 0.50234375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0118, | |
| "reward": 0.6545833051204681, | |
| "reward_std": 0.13637181371450424, | |
| "rewards/judge_tool_use": 0.7531250417232513, | |
| "rewards/judge_tool_use/std": 0.27382777631282806, | |
| "rewards/verify_correctness": 0.2604166641831398, | |
| "rewards/verify_correctness/std": 0.42997145652770996, | |
| "step": 200 | |
| }, | |
| { | |
| "clip_ratio": 0.0038763705641031267, | |
| "completion_length": 92.8402811686198, | |
| "epoch": 0.41, | |
| "grad_norm": 1.07948303423069, | |
| "kl": 0.391796875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0001, | |
| "reward": 0.7358333865801493, | |
| "reward_std": 0.14563809831937155, | |
| "rewards/judge_tool_use": 0.7756944497426351, | |
| "rewards/judge_tool_use/std": 0.3286278744538625, | |
| "rewards/verify_correctness": 0.5763888955116272, | |
| "rewards/verify_correctness/std": 0.4962470730145772, | |
| "step": 205 | |
| }, | |
| { | |
| "clip_ratio": 0.005373115837574005, | |
| "completion_length": 149.50000762939453, | |
| "epoch": 0.42, | |
| "grad_norm": 2.7654236631039004, | |
| "kl": 0.3029296875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.006, | |
| "reward": 0.6633333265781403, | |
| "reward_std": 0.14661981165409088, | |
| "rewards/judge_tool_use": 0.7197916805744171, | |
| "rewards/judge_tool_use/std": 0.3548731952905655, | |
| "rewards/verify_correctness": 0.4374999850988388, | |
| "rewards/verify_correctness/std": 0.49408672749996185, | |
| "step": 210 | |
| }, | |
| { | |
| "clip_ratio": 0.0028386844322085382, | |
| "completion_length": 108.20833333333333, | |
| "epoch": 0.43, | |
| "grad_norm": 2.009171248811446, | |
| "kl": 0.340234375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0052, | |
| "reward": 0.7725000381469727, | |
| "reward_std": 0.09887611617644627, | |
| "rewards/judge_tool_use": 0.8493055502573649, | |
| "rewards/judge_tool_use/std": 0.27945346136887866, | |
| "rewards/verify_correctness": 0.4652777910232544, | |
| "rewards/verify_correctness/std": 0.47633983691533405, | |
| "step": 215 | |
| }, | |
| { | |
| "clip_ratio": 0.0061874065548181536, | |
| "completion_length": 117.18750381469727, | |
| "epoch": 0.44, | |
| "grad_norm": 6.685116857049659, | |
| "kl": 0.394140625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0106, | |
| "reward": 0.6800000071525574, | |
| "reward_std": 0.19070632010698318, | |
| "rewards/judge_tool_use": 0.7510416507720947, | |
| "rewards/judge_tool_use/std": 0.3492521345615387, | |
| "rewards/verify_correctness": 0.3958333283662796, | |
| "rewards/verify_correctness/std": 0.4937349408864975, | |
| "step": 220 | |
| }, | |
| { | |
| "clip_ratio": 0.0023891992866992952, | |
| "completion_length": 105.63195037841797, | |
| "epoch": 0.45, | |
| "grad_norm": 0.639993744814138, | |
| "kl": 0.55, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0022, | |
| "reward": 0.7469444473584493, | |
| "reward_std": 0.11130555470784505, | |
| "rewards/judge_tool_use": 0.7756944298744202, | |
| "rewards/judge_tool_use/std": 0.3260296682516734, | |
| "rewards/verify_correctness": 0.6319444378217062, | |
| "rewards/verify_correctness/std": 0.47723113497098285, | |
| "step": 225 | |
| }, | |
| { | |
| "clip_ratio": 0.004893714282661677, | |
| "completion_length": 106.87500381469727, | |
| "epoch": 0.46, | |
| "grad_norm": 1.994854766362101, | |
| "kl": 0.412890625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0088, | |
| "reward": 0.7974999845027924, | |
| "reward_std": 0.11849463731050491, | |
| "rewards/judge_tool_use": 0.8666666746139526, | |
| "rewards/judge_tool_use/std": 0.22190550714731216, | |
| "rewards/verify_correctness": 0.5208333432674408, | |
| "rewards/verify_correctness/std": 0.49776600301265717, | |
| "step": 230 | |
| }, | |
| { | |
| "clip_ratio": 0.0027995842508971693, | |
| "completion_length": 98.25000254313152, | |
| "epoch": 0.47, | |
| "grad_norm": 0.8370010567585779, | |
| "kl": 0.578515625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0069, | |
| "reward": 0.7361111442248026, | |
| "reward_std": 0.08546942720810573, | |
| "rewards/judge_tool_use": 0.7708333333333334, | |
| "rewards/judge_tool_use/std": 0.3326546351114909, | |
| "rewards/verify_correctness": 0.597222218910853, | |
| "rewards/verify_correctness/std": 0.48818884293238324, | |
| "step": 235 | |
| }, | |
| { | |
| "clip_ratio": 0.0028807530645281075, | |
| "completion_length": 127.83333587646484, | |
| "epoch": 0.48, | |
| "grad_norm": 1.0910338182282877, | |
| "kl": 0.3333984375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0078, | |
| "reward": 0.8179166615009308, | |
| "reward_std": 0.10780513286590576, | |
| "rewards/judge_tool_use": 0.8739583194255829, | |
| "rewards/judge_tool_use/std": 0.2250625640153885, | |
| "rewards/verify_correctness": 0.5937499850988388, | |
| "rewards/verify_correctness/std": 0.48209600150585175, | |
| "step": 240 | |
| }, | |
| { | |
| "clip_ratio": 0.002781028486788273, | |
| "completion_length": 143.06944783528647, | |
| "epoch": 0.49, | |
| "grad_norm": 0.8178970498503804, | |
| "kl": 0.7380859375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0001, | |
| "reward": 0.6788889169692993, | |
| "reward_std": 0.10102842872341473, | |
| "rewards/judge_tool_use": 0.7027777830759684, | |
| "rewards/judge_tool_use/std": 0.3571552236874898, | |
| "rewards/verify_correctness": 0.5833333432674408, | |
| "rewards/verify_correctness/std": 0.4730796217918396, | |
| "step": 245 | |
| }, | |
| { | |
| "clip_ratio": 0.004690852202475071, | |
| "completion_length": 127.69792175292969, | |
| "epoch": 0.5, | |
| "grad_norm": 0.7032026926016068, | |
| "kl": 0.434375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0056, | |
| "reward": 0.7641666829586029, | |
| "reward_std": 0.1450955532491207, | |
| "rewards/judge_tool_use": 0.8354166746139526, | |
| "rewards/judge_tool_use/std": 0.2628621906042099, | |
| "rewards/verify_correctness": 0.4791666567325592, | |
| "rewards/verify_correctness/std": 0.5048523545265198, | |
| "step": 250 | |
| }, | |
| { | |
| "clip_ratio": 0.003919628076255321, | |
| "completion_length": 110.83333587646484, | |
| "epoch": 0.51, | |
| "grad_norm": 1.7992035669287993, | |
| "kl": 0.4609375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0157, | |
| "reward": 0.7619444529215494, | |
| "reward_std": 0.14088333398103714, | |
| "rewards/judge_tool_use": 0.8222222725550333, | |
| "rewards/judge_tool_use/std": 0.30879923701286316, | |
| "rewards/verify_correctness": 0.5208333333333334, | |
| "rewards/verify_correctness/std": 0.48915791511535645, | |
| "step": 255 | |
| }, | |
| { | |
| "clip_ratio": 0.005749301984906197, | |
| "completion_length": 112.65625, | |
| "epoch": 0.52, | |
| "grad_norm": 3.1299065923806113, | |
| "kl": 1.12109375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.001, | |
| "reward": 0.7679167091846466, | |
| "reward_std": 0.10637958347797394, | |
| "rewards/judge_tool_use": 0.8374999761581421, | |
| "rewards/judge_tool_use/std": 0.29924115538597107, | |
| "rewards/verify_correctness": 0.4895833283662796, | |
| "rewards/verify_correctness/std": 0.505071759223938, | |
| "step": 260 | |
| }, | |
| { | |
| "clip_ratio": 0.003776553086936474, | |
| "completion_length": 114.35417175292969, | |
| "epoch": 0.53, | |
| "grad_norm": 0.6562019375611381, | |
| "kl": 0.563671875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0019, | |
| "reward": 0.7005555629730225, | |
| "reward_std": 0.15318704272309938, | |
| "rewards/judge_tool_use": 0.7854166825612386, | |
| "rewards/judge_tool_use/std": 0.3270541826883952, | |
| "rewards/verify_correctness": 0.3611111218730609, | |
| "rewards/verify_correctness/std": 0.4106987814108531, | |
| "step": 265 | |
| }, | |
| { | |
| "clip_ratio": 0.0034743762575089932, | |
| "completion_length": 147.875, | |
| "epoch": 0.54, | |
| "grad_norm": 1.3887743953927447, | |
| "kl": 0.3212890625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0034, | |
| "reward": 0.8141666948795319, | |
| "reward_std": 0.12217209488153458, | |
| "rewards/judge_tool_use": 0.8979166746139526, | |
| "rewards/judge_tool_use/std": 0.16565020941197872, | |
| "rewards/verify_correctness": 0.4791666716337204, | |
| "rewards/verify_correctness/std": 0.49776598811149597, | |
| "step": 270 | |
| }, | |
| { | |
| "clip_ratio": 0.003613210655748844, | |
| "completion_length": 105.86805725097656, | |
| "epoch": 0.55, | |
| "grad_norm": 0.9278154163608585, | |
| "kl": 1.162890625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0073, | |
| "reward": 0.7766666611035665, | |
| "reward_std": 0.16950395703315735, | |
| "rewards/judge_tool_use": 0.8145833412806193, | |
| "rewards/judge_tool_use/std": 0.3171912630399068, | |
| "rewards/verify_correctness": 0.625, | |
| "rewards/verify_correctness/std": 0.48924607038497925, | |
| "step": 275 | |
| }, | |
| { | |
| "clip_ratio": 0.005631557852029801, | |
| "completion_length": 88.15625, | |
| "epoch": 0.56, | |
| "grad_norm": 0.49513702069352056, | |
| "kl": 0.349609375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0046, | |
| "reward": 0.9312500357627869, | |
| "reward_std": 0.0591347049921751, | |
| "rewards/judge_tool_use": 0.9583333134651184, | |
| "rewards/judge_tool_use/std": 0.11368590220808983, | |
| "rewards/verify_correctness": 0.8229166567325592, | |
| "rewards/verify_correctness/std": 0.38553228974342346, | |
| "step": 280 | |
| }, | |
| { | |
| "clip_ratio": 0.0010083256289362907, | |
| "completion_length": 111.0277811686198, | |
| "epoch": 0.57, | |
| "grad_norm": 0.9774895117172071, | |
| "kl": 0.299609375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0043, | |
| "reward": 0.8247222304344177, | |
| "reward_std": 0.12139152487119038, | |
| "rewards/judge_tool_use": 0.8694444497426351, | |
| "rewards/judge_tool_use/std": 0.2646999955177307, | |
| "rewards/verify_correctness": 0.6458333333333334, | |
| "rewards/verify_correctness/std": 0.4829867482185364, | |
| "step": 285 | |
| }, | |
| { | |
| "clip_ratio": 0.004534756857901811, | |
| "completion_length": 138.50000381469727, | |
| "epoch": 0.58, | |
| "grad_norm": 1.430617019290729, | |
| "kl": 0.3974609375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0021, | |
| "reward": 0.64000004529953, | |
| "reward_std": 0.129538856446743, | |
| "rewards/judge_tool_use": 0.7166666686534882, | |
| "rewards/judge_tool_use/std": 0.3585747927427292, | |
| "rewards/verify_correctness": 0.3333333358168602, | |
| "rewards/verify_correctness/std": 0.4409564733505249, | |
| "step": 290 | |
| }, | |
| { | |
| "clip_ratio": 0.003662421926856041, | |
| "completion_length": 118.62500254313152, | |
| "epoch": 0.59, | |
| "grad_norm": 0.779481967016939, | |
| "kl": 0.531640625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0063, | |
| "reward": 0.8033333619435629, | |
| "reward_std": 0.13420975704987845, | |
| "rewards/judge_tool_use": 0.8756944338480631, | |
| "rewards/judge_tool_use/std": 0.23473882675170898, | |
| "rewards/verify_correctness": 0.5138888855775198, | |
| "rewards/verify_correctness/std": 0.40222856402397156, | |
| "step": 295 | |
| }, | |
| { | |
| "clip_ratio": 0.0028870035894215107, | |
| "completion_length": 113.86458587646484, | |
| "epoch": 0.6, | |
| "grad_norm": 0.8439122527617477, | |
| "kl": 0.2859375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0006, | |
| "reward": 0.8366667032241821, | |
| "reward_std": 0.07533844374120235, | |
| "rewards/judge_tool_use": 0.8999999761581421, | |
| "rewards/judge_tool_use/std": 0.2058699131011963, | |
| "rewards/verify_correctness": 0.5833333432674408, | |
| "rewards/verify_correctness/std": 0.4908421188592911, | |
| "step": 300 | |
| }, | |
| { | |
| "clip_ratio": 0.0034239688422530888, | |
| "completion_length": 106.85416920979817, | |
| "epoch": 0.61, | |
| "grad_norm": 2.015558621646506, | |
| "kl": 0.965625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0182, | |
| "reward": 0.8475000063578287, | |
| "reward_std": 0.1549657036860784, | |
| "rewards/judge_tool_use": 0.8909722169240316, | |
| "rewards/judge_tool_use/std": 0.24881805976231894, | |
| "rewards/verify_correctness": 0.6736111044883728, | |
| "rewards/verify_correctness/std": 0.43851491808891296, | |
| "step": 305 | |
| }, | |
| { | |
| "clip_ratio": 0.0033080252818763257, | |
| "completion_length": 135.61459350585938, | |
| "epoch": 0.62, | |
| "grad_norm": 2.3005763258000322, | |
| "kl": 1.21640625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0275, | |
| "reward": 0.8000000417232513, | |
| "reward_std": 0.14312875270843506, | |
| "rewards/judge_tool_use": 0.8281249403953552, | |
| "rewards/judge_tool_use/std": 0.29270513355731964, | |
| "rewards/verify_correctness": 0.6875, | |
| "rewards/verify_correctness/std": 0.4634205251932144, | |
| "step": 310 | |
| }, | |
| { | |
| "clip_ratio": 0.0017321385443210603, | |
| "completion_length": 103.0763931274414, | |
| "epoch": 0.63, | |
| "grad_norm": 80.49583376546836, | |
| "kl": 4.99609375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0215, | |
| "reward": 0.9177778164545695, | |
| "reward_std": 0.07255559042096138, | |
| "rewards/judge_tool_use": 0.9527777433395386, | |
| "rewards/judge_tool_use/std": 0.12888848781585693, | |
| "rewards/verify_correctness": 0.7777777711550394, | |
| "rewards/verify_correctness/std": 0.40553895632425946, | |
| "step": 315 | |
| }, | |
| { | |
| "clip_ratio": 0.0032403219491243364, | |
| "completion_length": 97.76042175292969, | |
| "epoch": 0.64, | |
| "grad_norm": 2.6596636763815953, | |
| "kl": 1.287109375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.011, | |
| "reward": 0.8733333945274353, | |
| "reward_std": 0.08747303485870361, | |
| "rewards/judge_tool_use": 0.9145833253860474, | |
| "rewards/judge_tool_use/std": 0.20979993045330048, | |
| "rewards/verify_correctness": 0.7083333432674408, | |
| "rewards/verify_correctness/std": 0.45699402689933777, | |
| "step": 320 | |
| }, | |
| { | |
| "clip_ratio": 0.0033186592161655428, | |
| "completion_length": 117.89583841959636, | |
| "epoch": 0.65, | |
| "grad_norm": 1.1152329582646996, | |
| "kl": 0.5712890625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.02, | |
| "reward": 0.7755555907885233, | |
| "reward_std": 0.13340077300866446, | |
| "rewards/judge_tool_use": 0.7993055383364359, | |
| "rewards/judge_tool_use/std": 0.28823333978652954, | |
| "rewards/verify_correctness": 0.6805555621782938, | |
| "rewards/verify_correctness/std": 0.4147060612837474, | |
| "step": 325 | |
| }, | |
| { | |
| "clip_ratio": 0.0034143137745559216, | |
| "completion_length": 83.44791793823242, | |
| "epoch": 0.66, | |
| "grad_norm": 0.9507226451435283, | |
| "kl": 0.600390625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0022, | |
| "reward": 0.9216666221618652, | |
| "reward_std": 0.0693933442234993, | |
| "rewards/judge_tool_use": 0.9593749642372131, | |
| "rewards/judge_tool_use/std": 0.10937883332371712, | |
| "rewards/verify_correctness": 0.7708333134651184, | |
| "rewards/verify_correctness/std": 0.3960151970386505, | |
| "step": 330 | |
| }, | |
| { | |
| "clip_ratio": 0.00203145882114768, | |
| "completion_length": 143.0763956705729, | |
| "epoch": 0.67, | |
| "grad_norm": 1.94906459086153, | |
| "kl": 0.6546875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0089, | |
| "reward": 0.820555567741394, | |
| "reward_std": 0.10365821545322736, | |
| "rewards/judge_tool_use": 0.8833333253860474, | |
| "rewards/judge_tool_use/std": 0.21480779349803925, | |
| "rewards/verify_correctness": 0.5694444477558136, | |
| "rewards/verify_correctness/std": 0.49419113993644714, | |
| "step": 335 | |
| }, | |
| { | |
| "clip_ratio": 0.005462261289358139, | |
| "completion_length": 96.75, | |
| "epoch": 0.68, | |
| "grad_norm": 3.6005290276382365, | |
| "kl": 1.08046875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0146, | |
| "reward": 0.84333336353302, | |
| "reward_std": 0.10394849628210068, | |
| "rewards/judge_tool_use": 0.8562499582767487, | |
| "rewards/judge_tool_use/std": 0.28872600197792053, | |
| "rewards/verify_correctness": 0.7916666567325592, | |
| "rewards/verify_correctness/std": 0.3885633647441864, | |
| "step": 340 | |
| }, | |
| { | |
| "clip_ratio": 0.00261396998539567, | |
| "completion_length": 139.52083841959634, | |
| "epoch": 0.69, | |
| "grad_norm": 2.595391200258739, | |
| "kl": 0.625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0304, | |
| "reward": 0.7783333659172058, | |
| "reward_std": 0.16342870394388834, | |
| "rewards/judge_tool_use": 0.8201388716697693, | |
| "rewards/judge_tool_use/std": 0.27010903755823773, | |
| "rewards/verify_correctness": 0.6111111144224802, | |
| "rewards/verify_correctness/std": 0.46264639496803284, | |
| "step": 345 | |
| }, | |
| { | |
| "clip_ratio": 0.005096960160881281, | |
| "completion_length": 109.18750381469727, | |
| "epoch": 0.7, | |
| "grad_norm": 2.509616050978152, | |
| "kl": 1.11640625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0095, | |
| "reward": 0.8020833134651184, | |
| "reward_std": 0.1078796535730362, | |
| "rewards/judge_tool_use": 0.84375, | |
| "rewards/judge_tool_use/std": 0.23881300538778305, | |
| "rewards/verify_correctness": 0.6354166716337204, | |
| "rewards/verify_correctness/std": 0.4489881992340088, | |
| "step": 350 | |
| }, | |
| { | |
| "clip_ratio": 0.0030191333964467047, | |
| "completion_length": 119.64583587646484, | |
| "epoch": 0.71, | |
| "grad_norm": 1.9232078822128884, | |
| "kl": 0.651171875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0193, | |
| "reward": 0.8113889296849569, | |
| "reward_std": 0.09605963776508968, | |
| "rewards/judge_tool_use": 0.8597222169240316, | |
| "rewards/judge_tool_use/std": 0.23875926931699118, | |
| "rewards/verify_correctness": 0.6180555621782938, | |
| "rewards/verify_correctness/std": 0.41723007957140607, | |
| "step": 355 | |
| }, | |
| { | |
| "clip_ratio": 0.0040064876899123195, | |
| "completion_length": 107.82292175292969, | |
| "epoch": 0.72, | |
| "grad_norm": 0.8384093480805856, | |
| "kl": 0.59375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0015, | |
| "reward": 0.8174999952316284, | |
| "reward_std": 0.07755549252033234, | |
| "rewards/judge_tool_use": 0.8760416805744171, | |
| "rewards/judge_tool_use/std": 0.21128008142113686, | |
| "rewards/verify_correctness": 0.5833333432674408, | |
| "rewards/verify_correctness/std": 0.4908421188592911, | |
| "step": 360 | |
| }, | |
| { | |
| "clip_ratio": 0.003883673809468746, | |
| "completion_length": 118.04167175292969, | |
| "epoch": 0.73, | |
| "grad_norm": 3.6270547715534547, | |
| "kl": 0.630859375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0123, | |
| "reward": 0.8119444648424784, | |
| "reward_std": 0.10799338171879451, | |
| "rewards/judge_tool_use": 0.8708332975705465, | |
| "rewards/judge_tool_use/std": 0.24002541601657867, | |
| "rewards/verify_correctness": 0.5763889054457346, | |
| "rewards/verify_correctness/std": 0.49137688676516217, | |
| "step": 365 | |
| }, | |
| { | |
| "clip_ratio": 0.002587809320539236, | |
| "completion_length": 120.89583587646484, | |
| "epoch": 0.74, | |
| "grad_norm": 2.313493437394313, | |
| "kl": 0.56640625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0077, | |
| "reward": 0.8029167056083679, | |
| "reward_std": 0.12100144848227501, | |
| "rewards/judge_tool_use": 0.8760416805744171, | |
| "rewards/judge_tool_use/std": 0.26075808703899384, | |
| "rewards/verify_correctness": 0.5104166716337204, | |
| "rewards/verify_correctness/std": 0.47240525484085083, | |
| "step": 370 | |
| }, | |
| { | |
| "clip_ratio": 0.0027662038803100584, | |
| "completion_length": 130.86805979410806, | |
| "epoch": 0.75, | |
| "grad_norm": 1.4291418174291879, | |
| "kl": 0.58203125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0109, | |
| "reward": 0.859166661898295, | |
| "reward_std": 0.13911124070485434, | |
| "rewards/judge_tool_use": 0.8986111283302307, | |
| "rewards/judge_tool_use/std": 0.21422065049409866, | |
| "rewards/verify_correctness": 0.7013888756434122, | |
| "rewards/verify_correctness/std": 0.4320597251256307, | |
| "step": 375 | |
| }, | |
| { | |
| "clip_ratio": 0.004125702893361449, | |
| "completion_length": 102.20833587646484, | |
| "epoch": 0.76, | |
| "grad_norm": 0.8091042916650757, | |
| "kl": 0.345703125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.001, | |
| "reward": 0.8775000274181366, | |
| "reward_std": 0.07746638730168343, | |
| "rewards/judge_tool_use": 0.925000011920929, | |
| "rewards/judge_tool_use/std": 0.19297337904572487, | |
| "rewards/verify_correctness": 0.6875, | |
| "rewards/verify_correctness/std": 0.4594741761684418, | |
| "step": 380 | |
| }, | |
| { | |
| "clip_ratio": 0.0013461930211633444, | |
| "completion_length": 116.84722391764323, | |
| "epoch": 0.77, | |
| "grad_norm": 0.8881903330505401, | |
| "kl": 3.155859375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0164, | |
| "reward": 0.8816666801770529, | |
| "reward_std": 0.07364911710222562, | |
| "rewards/judge_tool_use": 0.9284722208976746, | |
| "rewards/judge_tool_use/std": 0.1873454600572586, | |
| "rewards/verify_correctness": 0.694444457689921, | |
| "rewards/verify_correctness/std": 0.4634603758653005, | |
| "step": 385 | |
| }, | |
| { | |
| "clip_ratio": 0.0035483752377331256, | |
| "completion_length": 113.29166793823242, | |
| "epoch": 0.78, | |
| "grad_norm": 1.9115550749698071, | |
| "kl": 0.36015625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.006, | |
| "reward": 0.9200000166893005, | |
| "reward_std": 0.07504565827548504, | |
| "rewards/judge_tool_use": 0.9677083492279053, | |
| "rewards/judge_tool_use/std": 0.0825247336179018, | |
| "rewards/verify_correctness": 0.7291666865348816, | |
| "rewards/verify_correctness/std": 0.44340358674526215, | |
| "step": 390 | |
| }, | |
| { | |
| "clip_ratio": 0.0033025143668055533, | |
| "completion_length": 118.38194529215495, | |
| "epoch": 0.79, | |
| "grad_norm": 0.6918361378607686, | |
| "kl": 0.459375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0016, | |
| "reward": 0.8894444505373637, | |
| "reward_std": 0.09632904827594757, | |
| "rewards/judge_tool_use": 0.9416666428248087, | |
| "rewards/judge_tool_use/std": 0.13143713772296906, | |
| "rewards/verify_correctness": 0.6805555621782938, | |
| "rewards/verify_correctness/std": 0.46167699495951336, | |
| "step": 395 | |
| }, | |
| { | |
| "clip_ratio": 0.003618121985346079, | |
| "completion_length": 118.16666793823242, | |
| "epoch": 0.8, | |
| "grad_norm": 1.7518446245486312, | |
| "kl": 0.746484375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0075, | |
| "reward": 0.8358333706855774, | |
| "reward_std": 0.1284572072327137, | |
| "rewards/judge_tool_use": 0.8625000417232513, | |
| "rewards/judge_tool_use/std": 0.27199896425008774, | |
| "rewards/verify_correctness": 0.7291666567325592, | |
| "rewards/verify_correctness/std": 0.4388819485902786, | |
| "step": 400 | |
| }, | |
| { | |
| "clip_ratio": 0.003691716957837343, | |
| "completion_length": 110.00694783528645, | |
| "epoch": 0.81, | |
| "grad_norm": 0.8896421929804095, | |
| "kl": 0.3625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0004, | |
| "reward": 0.8361111481984457, | |
| "reward_std": 0.08633811896046002, | |
| "rewards/judge_tool_use": 0.8958333333333334, | |
| "rewards/judge_tool_use/std": 0.1966320425271988, | |
| "rewards/verify_correctness": 0.5972222089767456, | |
| "rewards/verify_correctness/std": 0.48761820793151855, | |
| "step": 405 | |
| }, | |
| { | |
| "clip_ratio": 0.003539162687957287, | |
| "completion_length": 114.38542175292969, | |
| "epoch": 0.82, | |
| "grad_norm": 0.8364119133705628, | |
| "kl": 0.416796875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0046, | |
| "reward": 0.8329166769981384, | |
| "reward_std": 0.07315276563167572, | |
| "rewards/judge_tool_use": 0.8770833611488342, | |
| "rewards/judge_tool_use/std": 0.2352844625711441, | |
| "rewards/verify_correctness": 0.65625, | |
| "rewards/verify_correctness/std": 0.44986698031425476, | |
| "step": 410 | |
| }, | |
| { | |
| "clip_ratio": 0.001082425657659769, | |
| "completion_length": 105.50694783528645, | |
| "epoch": 0.83, | |
| "grad_norm": 1.1930359747277361, | |
| "kl": 0.505078125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0135, | |
| "reward": 0.8663889169692993, | |
| "reward_std": 0.08535568416118622, | |
| "rewards/judge_tool_use": 0.9388888875643412, | |
| "rewards/judge_tool_use/std": 0.1443981478611628, | |
| "rewards/verify_correctness": 0.5763888855775198, | |
| "rewards/verify_correctness/std": 0.4668743809064229, | |
| "step": 415 | |
| }, | |
| { | |
| "clip_ratio": 0.0036332331597805022, | |
| "completion_length": 91.27083587646484, | |
| "epoch": 0.84, | |
| "grad_norm": 1.0418583888118527, | |
| "kl": 0.48046875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0021, | |
| "reward": 0.8641667366027832, | |
| "reward_std": 0.08411096408963203, | |
| "rewards/judge_tool_use": 0.9135416746139526, | |
| "rewards/judge_tool_use/std": 0.1935550421476364, | |
| "rewards/verify_correctness": 0.6666666567325592, | |
| "rewards/verify_correctness/std": 0.44964754581451416, | |
| "step": 420 | |
| }, | |
| { | |
| "clip_ratio": 0.0031434688717126846, | |
| "completion_length": 136.47222900390625, | |
| "epoch": 0.85, | |
| "grad_norm": 9.602708864091571, | |
| "kl": 0.8896484375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0095, | |
| "reward": 0.8213888804117838, | |
| "reward_std": 0.10578912993272145, | |
| "rewards/judge_tool_use": 0.8791666825612386, | |
| "rewards/judge_tool_use/std": 0.21359455088774362, | |
| "rewards/verify_correctness": 0.5902777711550394, | |
| "rewards/verify_correctness/std": 0.48865292469660443, | |
| "step": 425 | |
| }, | |
| { | |
| "clip_ratio": 0.006819223985075951, | |
| "completion_length": 121.85417175292969, | |
| "epoch": 0.86, | |
| "grad_norm": 0.9453508121139471, | |
| "kl": 0.821875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0144, | |
| "reward": 0.7583333849906921, | |
| "reward_std": 0.15919847041368484, | |
| "rewards/judge_tool_use": 0.8177083134651184, | |
| "rewards/judge_tool_use/std": 0.27655909955501556, | |
| "rewards/verify_correctness": 0.5208333283662796, | |
| "rewards/verify_correctness/std": 0.5030901730060577, | |
| "step": 430 | |
| }, | |
| { | |
| "clip_ratio": 0.005021798610687256, | |
| "completion_length": 117.34722391764323, | |
| "epoch": 0.87, | |
| "grad_norm": 0.9574949482570072, | |
| "kl": 2.016796875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0024, | |
| "reward": 0.8338889280954996, | |
| "reward_std": 0.11587712913751602, | |
| "rewards/judge_tool_use": 0.868749996026357, | |
| "rewards/judge_tool_use/std": 0.24017359067996344, | |
| "rewards/verify_correctness": 0.6944444378217062, | |
| "rewards/verify_correctness/std": 0.46080262462298077, | |
| "step": 435 | |
| }, | |
| { | |
| "clip_ratio": 0.003334458451718092, | |
| "completion_length": 102.16667175292969, | |
| "epoch": 0.88, | |
| "grad_norm": 1.284555858192564, | |
| "kl": 0.491015625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0001, | |
| "reward": 0.8824999928474426, | |
| "reward_std": 0.07597887888550758, | |
| "rewards/judge_tool_use": 0.9364582896232605, | |
| "rewards/judge_tool_use/std": 0.18419649451971054, | |
| "rewards/verify_correctness": 0.6666666865348816, | |
| "rewards/verify_correctness/std": 0.47639307379722595, | |
| "step": 440 | |
| }, | |
| { | |
| "clip_ratio": 0.0027906086295843126, | |
| "completion_length": 98.38194783528645, | |
| "epoch": 0.89, | |
| "grad_norm": 0.4954906916589266, | |
| "kl": 0.650390625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0002, | |
| "reward": 0.8658333619435629, | |
| "reward_std": 0.07699522500236829, | |
| "rewards/judge_tool_use": 0.9381944338480631, | |
| "rewards/judge_tool_use/std": 0.132844394693772, | |
| "rewards/verify_correctness": 0.5763888955116272, | |
| "rewards/verify_correctness/std": 0.47234441836675006, | |
| "step": 445 | |
| }, | |
| { | |
| "clip_ratio": 0.003258515940979123, | |
| "completion_length": 87.12500381469727, | |
| "epoch": 0.9, | |
| "grad_norm": 0.8195724812478982, | |
| "kl": 0.283984375, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0011, | |
| "reward": 0.8491666615009308, | |
| "reward_std": 0.10520510375499725, | |
| "rewards/judge_tool_use": 0.9000000357627869, | |
| "rewards/judge_tool_use/std": 0.19222086668014526, | |
| "rewards/verify_correctness": 0.6458333283662796, | |
| "rewards/verify_correctness/std": 0.44007769227027893, | |
| "step": 450 | |
| }, | |
| { | |
| "clip_ratio": 0.003890740033239126, | |
| "completion_length": 109.83333587646484, | |
| "epoch": 0.91, | |
| "grad_norm": 0.7607892651828614, | |
| "kl": 0.4947265625, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0018, | |
| "reward": 0.8672222892443339, | |
| "reward_std": 0.11101956913868587, | |
| "rewards/judge_tool_use": 0.8999999761581421, | |
| "rewards/judge_tool_use/std": 0.19080724567174911, | |
| "rewards/verify_correctness": 0.7361111243565878, | |
| "rewards/verify_correctness/std": 0.43669142325719196, | |
| "step": 455 | |
| }, | |
| { | |
| "clip_ratio": 0.003977376967668533, | |
| "completion_length": 104.58333969116211, | |
| "epoch": 0.92, | |
| "grad_norm": 0.31311068474329323, | |
| "kl": 0.4921875, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0072, | |
| "reward": 0.8212500214576721, | |
| "reward_std": 0.08914041426032782, | |
| "rewards/judge_tool_use": 0.8364583849906921, | |
| "rewards/judge_tool_use/std": 0.2007633000612259, | |
| "rewards/verify_correctness": 0.7604166567325592, | |
| "rewards/verify_correctness/std": 0.3714234232902527, | |
| "step": 460 | |
| }, | |
| { | |
| "clip_ratio": 0.0033224607817828656, | |
| "completion_length": 109.66666920979817, | |
| "epoch": 0.93, | |
| "grad_norm": 0.9597186134421405, | |
| "kl": 0.775, | |
| "learning_rate": 1e-06, | |
| "loss": 0.003, | |
| "reward": 0.7788889010747274, | |
| "reward_std": 0.07402739052971204, | |
| "rewards/judge_tool_use": 0.8104166587193807, | |
| "rewards/judge_tool_use/std": 0.24389235799511275, | |
| "rewards/verify_correctness": 0.652777781089147, | |
| "rewards/verify_correctness/std": 0.3849090288082759, | |
| "step": 465 | |
| }, | |
| { | |
| "clip_ratio": 0.004331017844378948, | |
| "completion_length": 128.03125762939453, | |
| "epoch": 0.94, | |
| "grad_norm": 0.6604515564011074, | |
| "kl": 0.440234375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.002, | |
| "reward": 0.8320833444595337, | |
| "reward_std": 0.11683276295661926, | |
| "rewards/judge_tool_use": 0.8968749940395355, | |
| "rewards/judge_tool_use/std": 0.20512499660253525, | |
| "rewards/verify_correctness": 0.5729166567325592, | |
| "rewards/verify_correctness/std": 0.49430613219738007, | |
| "step": 470 | |
| }, | |
| { | |
| "clip_ratio": 0.003034461010247469, | |
| "completion_length": 129.48611704508463, | |
| "epoch": 0.95, | |
| "grad_norm": 1.7164077020687818, | |
| "kl": 0.575390625, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0071, | |
| "reward": 0.7255556186040243, | |
| "reward_std": 0.09651681656638782, | |
| "rewards/judge_tool_use": 0.7993055780728658, | |
| "rewards/judge_tool_use/std": 0.2533370802799861, | |
| "rewards/verify_correctness": 0.4305555572112401, | |
| "rewards/verify_correctness/std": 0.41316067179044086, | |
| "step": 475 | |
| }, | |
| { | |
| "clip_ratio": 0.004247188335284591, | |
| "completion_length": 90.69791793823242, | |
| "epoch": 0.96, | |
| "grad_norm": 0.26610601551372964, | |
| "kl": 0.3984375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0013, | |
| "reward": 0.8808333277702332, | |
| "reward_std": 0.07401393819600344, | |
| "rewards/judge_tool_use": 0.9291666746139526, | |
| "rewards/judge_tool_use/std": 0.12290690839290619, | |
| "rewards/verify_correctness": 0.6875, | |
| "rewards/verify_correctness/std": 0.44007769227027893, | |
| "step": 480 | |
| }, | |
| { | |
| "clip_ratio": 0.0029933893121778963, | |
| "completion_length": 100.36111450195312, | |
| "epoch": 0.97, | |
| "grad_norm": 0.1513677218763824, | |
| "kl": 0.52734375, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0008, | |
| "reward": 0.9199999968210856, | |
| "reward_std": 0.05473165214061737, | |
| "rewards/judge_tool_use": 0.9555555582046509, | |
| "rewards/judge_tool_use/std": 0.0996720480422179, | |
| "rewards/verify_correctness": 0.7777777711550394, | |
| "rewards/verify_correctness/std": 0.3172461787859599, | |
| "step": 485 | |
| }, | |
| { | |
| "clip_ratio": 0.00557863749563694, | |
| "completion_length": 111.61458587646484, | |
| "epoch": 0.98, | |
| "grad_norm": 1.166430172051773, | |
| "kl": 0.385546875, | |
| "learning_rate": 1e-06, | |
| "loss": -0.0025, | |
| "reward": 0.7954167127609253, | |
| "reward_std": 0.1558975875377655, | |
| "rewards/judge_tool_use": 0.8562500476837158, | |
| "rewards/judge_tool_use/std": 0.24550338834524155, | |
| "rewards/verify_correctness": 0.5520833283662796, | |
| "rewards/verify_correctness/std": 0.46045805513858795, | |
| "step": 490 | |
| }, | |
| { | |
| "clip_ratio": 0.0024958730675280092, | |
| "completion_length": 118.20833841959636, | |
| "epoch": 0.99, | |
| "grad_norm": 0.39020950446372304, | |
| "kl": 0.2814453125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0019, | |
| "reward": 0.7777777910232544, | |
| "reward_std": 0.0677791138490041, | |
| "rewards/judge_tool_use": 0.8055555621782938, | |
| "rewards/judge_tool_use/std": 0.2596944694717725, | |
| "rewards/verify_correctness": 0.6666666666666666, | |
| "rewards/verify_correctness/std": 0.4650210638840993, | |
| "step": 495 | |
| }, | |
| { | |
| "clip_ratio": 0.004371361434459686, | |
| "completion_length": 109.94791793823242, | |
| "epoch": 1.0, | |
| "grad_norm": 1.0635984322785825, | |
| "kl": 0.35078125, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0043, | |
| "reward": 0.8491666913032532, | |
| "reward_std": 0.08863747119903564, | |
| "rewards/judge_tool_use": 0.8895833194255829, | |
| "rewards/judge_tool_use/std": 0.23481010645627975, | |
| "rewards/verify_correctness": 0.6875, | |
| "rewards/verify_correctness/std": 0.46786636114120483, | |
| "step": 500 | |
| } | |
| ], | |
| "logging_steps": 5, | |
| "max_steps": 500, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 100, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 8, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |