| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.0, | |
| "eval_steps": 500, | |
| "global_step": 500, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.002, | |
| "grad_norm": 0.0, | |
| "kl": 0.0, | |
| "learning_rate": 2e-08, | |
| "loss": 0.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 1 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.004, | |
| "grad_norm": 0.6894801259040833, | |
| "kl": 0.0, | |
| "learning_rate": 4e-08, | |
| "loss": 0.0, | |
| "reward": 0.5625, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.0625, | |
| "step": 2 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1845.75, | |
| "epoch": 0.006, | |
| "grad_norm": 0.0036829786840826273, | |
| "kl": 4.696846008300781e-05, | |
| "learning_rate": 6e-08, | |
| "loss": 0.0, | |
| "reward": 0.0, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.0, | |
| "step": 3 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.008, | |
| "grad_norm": 0.0038558689411729574, | |
| "kl": 9.775161743164062e-05, | |
| "learning_rate": 8e-08, | |
| "loss": 0.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 4 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.01, | |
| "grad_norm": 0.7485816478729248, | |
| "kl": 0.00010919570922851562, | |
| "learning_rate": 1e-07, | |
| "loss": 0.0, | |
| "reward": 0.0625, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.0625, | |
| "step": 5 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1985.5, | |
| "epoch": 0.012, | |
| "grad_norm": 0.7458791732788086, | |
| "kl": 7.796287536621094e-05, | |
| "learning_rate": 1.2e-07, | |
| "loss": -0.023, | |
| "reward": 0.0625, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.0625, | |
| "step": 6 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.014, | |
| "grad_norm": 0.003527791704982519, | |
| "kl": 6.628036499023438e-05, | |
| "learning_rate": 1.4e-07, | |
| "loss": 0.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 7 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.016, | |
| "grad_norm": 0.8426324129104614, | |
| "kl": 7.104873657226562e-05, | |
| "learning_rate": 1.6e-07, | |
| "loss": 0.0, | |
| "reward": 0.0625, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.0625, | |
| "step": 8 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.018, | |
| "grad_norm": 0.7430940270423889, | |
| "kl": 9.5367431640625e-05, | |
| "learning_rate": 1.8e-07, | |
| "loss": 0.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 9 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.02, | |
| "grad_norm": 0.003660305170342326, | |
| "kl": 5.626678466796875e-05, | |
| "learning_rate": 2e-07, | |
| "loss": 0.0, | |
| "reward": 0.0, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.0, | |
| "step": 10 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.022, | |
| "grad_norm": 0.9972493648529053, | |
| "kl": 0.0001544952392578125, | |
| "learning_rate": 2.1999999999999998e-07, | |
| "loss": 0.0, | |
| "reward": 0.0625, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.0625, | |
| "step": 11 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.024, | |
| "grad_norm": 0.003556522075086832, | |
| "kl": 6.723403930664062e-05, | |
| "learning_rate": 2.4e-07, | |
| "loss": 0.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 12 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1682.25, | |
| "epoch": 0.026, | |
| "grad_norm": 0.0050394581630826, | |
| "kl": 8.845329284667969e-05, | |
| "learning_rate": 2.6e-07, | |
| "loss": 0.0, | |
| "reward": 0.0, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.0, | |
| "step": 13 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.028, | |
| "grad_norm": 0.004075606819242239, | |
| "kl": 0.0001049041748046875, | |
| "learning_rate": 2.8e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 14 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.03, | |
| "grad_norm": 0.6532822251319885, | |
| "kl": 7.2479248046875e-05, | |
| "learning_rate": 3e-07, | |
| "loss": 0.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 15 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.032, | |
| "grad_norm": 0.7762453556060791, | |
| "kl": 0.00011777877807617188, | |
| "learning_rate": 3.2e-07, | |
| "loss": 0.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 16 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.034, | |
| "grad_norm": 0.7456417679786682, | |
| "kl": 8.58306884765625e-05, | |
| "learning_rate": 3.4000000000000003e-07, | |
| "loss": 0.0, | |
| "reward": 0.0625, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.0625, | |
| "step": 17 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1770.5, | |
| "epoch": 0.036, | |
| "grad_norm": 0.0034950117114931345, | |
| "kl": 3.5762786865234375e-05, | |
| "learning_rate": 3.6e-07, | |
| "loss": 0.0, | |
| "reward": 0.0, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.0, | |
| "step": 18 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.038, | |
| "grad_norm": 0.003600472817197442, | |
| "kl": 6.866455078125e-05, | |
| "learning_rate": 3.7999999999999996e-07, | |
| "loss": 0.0, | |
| "reward": 0.0, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.0, | |
| "step": 19 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1856.25, | |
| "epoch": 0.04, | |
| "grad_norm": 0.8460555076599121, | |
| "kl": 8.106231689453125e-05, | |
| "learning_rate": 4e-07, | |
| "loss": 0.0814, | |
| "reward": 0.0625, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.0625, | |
| "step": 20 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.042, | |
| "grad_norm": 0.0038911281153559685, | |
| "kl": 0.0001068115234375, | |
| "learning_rate": 4.1999999999999995e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 21 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.044, | |
| "grad_norm": 0.7139887809753418, | |
| "kl": 0.0001125335693359375, | |
| "learning_rate": 4.3999999999999997e-07, | |
| "loss": 0.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 22 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1855.25, | |
| "epoch": 0.046, | |
| "grad_norm": 0.005301118828356266, | |
| "kl": 5.6862831115722656e-05, | |
| "learning_rate": 4.6e-07, | |
| "loss": 0.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 23 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.048, | |
| "grad_norm": 0.0036839963868260384, | |
| "kl": 7.62939453125e-05, | |
| "learning_rate": 4.8e-07, | |
| "loss": 0.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 24 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.05, | |
| "grad_norm": 0.003639479400590062, | |
| "kl": 8.678436279296875e-05, | |
| "learning_rate": 5e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 25 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.052, | |
| "grad_norm": 0.8071563243865967, | |
| "kl": 8.535385131835938e-05, | |
| "learning_rate": 5.2e-07, | |
| "loss": 0.0, | |
| "reward": 0.0625, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.0625, | |
| "step": 26 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.054, | |
| "grad_norm": 0.0037776094395667315, | |
| "kl": 0.00011014938354492188, | |
| "learning_rate": 5.4e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 27 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1608.75, | |
| "epoch": 0.056, | |
| "grad_norm": 0.6625211834907532, | |
| "kl": 4.398822784423828e-05, | |
| "learning_rate": 5.6e-07, | |
| "loss": 0.0, | |
| "reward": 0.0625, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.0625, | |
| "step": 28 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1841.25, | |
| "epoch": 0.058, | |
| "grad_norm": 0.0036761562805622816, | |
| "kl": 8.678436279296875e-05, | |
| "learning_rate": 5.8e-07, | |
| "loss": 0.0, | |
| "reward": 0.0, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.0, | |
| "step": 29 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.06, | |
| "grad_norm": 0.0033802345860749483, | |
| "kl": 5.7697296142578125e-05, | |
| "learning_rate": 6e-07, | |
| "loss": 0.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 30 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.062, | |
| "grad_norm": 0.003240807680413127, | |
| "kl": 6.29425048828125e-05, | |
| "learning_rate": 6.2e-07, | |
| "loss": 0.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 31 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.064, | |
| "grad_norm": 0.0036673492286354303, | |
| "kl": 7.152557373046875e-05, | |
| "learning_rate": 6.4e-07, | |
| "loss": 0.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 32 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1735.75, | |
| "epoch": 0.066, | |
| "grad_norm": 0.8675172328948975, | |
| "kl": 2.682209014892578e-05, | |
| "learning_rate": 6.6e-07, | |
| "loss": -0.0631, | |
| "reward": 0.0625, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.0625, | |
| "step": 33 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.068, | |
| "grad_norm": 0.8184774518013, | |
| "kl": 9.775161743164062e-05, | |
| "learning_rate": 6.800000000000001e-07, | |
| "loss": 0.0, | |
| "reward": 0.0625, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.0625, | |
| "step": 34 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.07, | |
| "grad_norm": 0.003660478862002492, | |
| "kl": 0.00010251998901367188, | |
| "learning_rate": 7e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 35 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.072, | |
| "grad_norm": 0.004034126177430153, | |
| "kl": 6.4849853515625e-05, | |
| "learning_rate": 7.2e-07, | |
| "loss": 0.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 36 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.074, | |
| "grad_norm": 0.7589001655578613, | |
| "kl": 0.00010824203491210938, | |
| "learning_rate": 7.4e-07, | |
| "loss": 0.0, | |
| "reward": 0.0625, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.0625, | |
| "step": 37 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.076, | |
| "grad_norm": 0.6517212986946106, | |
| "kl": 8.20159912109375e-05, | |
| "learning_rate": 7.599999999999999e-07, | |
| "loss": 0.0, | |
| "reward": 0.0625, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.0625, | |
| "step": 38 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.078, | |
| "grad_norm": 0.8392653465270996, | |
| "kl": 0.0001430511474609375, | |
| "learning_rate": 7.799999999999999e-07, | |
| "loss": 0.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 39 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.08, | |
| "grad_norm": 0.990591824054718, | |
| "kl": 0.0001125335693359375, | |
| "learning_rate": 8e-07, | |
| "loss": 0.0, | |
| "reward": 0.125, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 40 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.082, | |
| "grad_norm": 0.7600575685501099, | |
| "kl": 0.00014066696166992188, | |
| "learning_rate": 8.199999999999999e-07, | |
| "loss": 0.0, | |
| "reward": 0.0625, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.0625, | |
| "step": 41 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.084, | |
| "grad_norm": 0.004982170183211565, | |
| "kl": 0.00013589859008789062, | |
| "learning_rate": 8.399999999999999e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 42 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.086, | |
| "grad_norm": 0.6588786840438843, | |
| "kl": 0.00019502639770507812, | |
| "learning_rate": 8.599999999999999e-07, | |
| "loss": 0.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 43 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.088, | |
| "grad_norm": 0.876471996307373, | |
| "kl": 0.00020933151245117188, | |
| "learning_rate": 8.799999999999999e-07, | |
| "loss": 0.0, | |
| "reward": 0.0625, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.0625, | |
| "step": 44 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1857.0, | |
| "epoch": 0.09, | |
| "grad_norm": 0.007592031732201576, | |
| "kl": 0.00015783309936523438, | |
| "learning_rate": 9e-07, | |
| "loss": 0.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 45 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.092, | |
| "grad_norm": 0.649493932723999, | |
| "kl": 0.00015401840209960938, | |
| "learning_rate": 9.2e-07, | |
| "loss": 0.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 46 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.094, | |
| "grad_norm": 0.0060058352537453175, | |
| "kl": 0.00024318695068359375, | |
| "learning_rate": 9.399999999999999e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 47 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1703.5, | |
| "epoch": 0.096, | |
| "grad_norm": 0.7367803454399109, | |
| "kl": 0.00028061866760253906, | |
| "learning_rate": 9.6e-07, | |
| "loss": 0.0, | |
| "reward": 0.0625, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.0625, | |
| "step": 48 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.098, | |
| "grad_norm": 0.7102321982383728, | |
| "kl": 0.0003223419189453125, | |
| "learning_rate": 9.8e-07, | |
| "loss": 0.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 49 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.1, | |
| "grad_norm": 0.5775962471961975, | |
| "kl": 0.00010824203491210938, | |
| "learning_rate": 1e-06, | |
| "loss": 0.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 50 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.102, | |
| "grad_norm": 0.0037765211891382933, | |
| "kl": 7.62939453125e-05, | |
| "learning_rate": 9.999890338174275e-07, | |
| "loss": 0.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 51 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1991.5, | |
| "epoch": 0.104, | |
| "grad_norm": 0.8791236281394958, | |
| "kl": 0.00046634674072265625, | |
| "learning_rate": 9.999561358041868e-07, | |
| "loss": 0.0, | |
| "reward": 0.0625, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.0625, | |
| "step": 52 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.106, | |
| "grad_norm": 0.006120009813457727, | |
| "kl": 0.0004935264587402344, | |
| "learning_rate": 9.999013075636804e-07, | |
| "loss": 0.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 53 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.108, | |
| "grad_norm": 0.006084068212658167, | |
| "kl": 0.0002951622009277344, | |
| "learning_rate": 9.998245517681593e-07, | |
| "loss": 0.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 54 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1732.5, | |
| "epoch": 0.11, | |
| "grad_norm": 0.008206584490835667, | |
| "kl": 0.00015926361083984375, | |
| "learning_rate": 9.997258721585931e-07, | |
| "loss": 0.0, | |
| "reward": 0.5, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.0, | |
| "step": 55 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2031.5, | |
| "epoch": 0.112, | |
| "grad_norm": 0.06737767159938812, | |
| "kl": 0.0014257431030273438, | |
| "learning_rate": 9.996052735444862e-07, | |
| "loss": 0.0001, | |
| "reward": 0.0, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.0, | |
| "step": 56 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.114, | |
| "grad_norm": 0.6304325461387634, | |
| "kl": 0.0003185272216796875, | |
| "learning_rate": 9.994627618036452e-07, | |
| "loss": 0.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 57 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.116, | |
| "grad_norm": 0.6972232460975647, | |
| "kl": 0.000560760498046875, | |
| "learning_rate": 9.992983438818915e-07, | |
| "loss": 0.0, | |
| "reward": 0.0625, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.0625, | |
| "step": 58 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.118, | |
| "grad_norm": 0.026448842138051987, | |
| "kl": 0.0009822845458984375, | |
| "learning_rate": 9.991120277927223e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 59 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.12, | |
| "grad_norm": 0.006011964753270149, | |
| "kl": 0.0007162094116210938, | |
| "learning_rate": 9.989038226169207e-07, | |
| "loss": 0.0, | |
| "reward": 0.75, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 60 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.122, | |
| "grad_norm": 0.6680987477302551, | |
| "kl": 0.0005612373352050781, | |
| "learning_rate": 9.98673738502114e-07, | |
| "loss": 0.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 61 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1933.75, | |
| "epoch": 0.124, | |
| "grad_norm": 0.9241335988044739, | |
| "kl": 0.0006771087646484375, | |
| "learning_rate": 9.98421786662277e-07, | |
| "loss": -0.0443, | |
| "reward": 0.6875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 62 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.126, | |
| "grad_norm": 0.009985378012061119, | |
| "kl": 0.0005092620849609375, | |
| "learning_rate": 9.981479793771866e-07, | |
| "loss": 0.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 63 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.128, | |
| "grad_norm": 0.03178563341498375, | |
| "kl": 0.0007648468017578125, | |
| "learning_rate": 9.97852329991824e-07, | |
| "loss": 0.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 64 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1685.75, | |
| "epoch": 0.13, | |
| "grad_norm": 0.9064852595329285, | |
| "kl": 0.00341796875, | |
| "learning_rate": 9.975348529157229e-07, | |
| "loss": -0.0454, | |
| "reward": 0.0625, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.0625, | |
| "step": 65 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.132, | |
| "grad_norm": 0.7919860482215881, | |
| "kl": 0.001010894775390625, | |
| "learning_rate": 9.971955636222684e-07, | |
| "loss": 0.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 66 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1887.25, | |
| "epoch": 0.134, | |
| "grad_norm": 0.8760141134262085, | |
| "kl": 0.000766754150390625, | |
| "learning_rate": 9.968344786479415e-07, | |
| "loss": 0.0659, | |
| "reward": 0.375, | |
| "reward_std": 0.3535533845424652, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.375, | |
| "step": 67 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.136, | |
| "grad_norm": 0.004815725143998861, | |
| "kl": 0.0005044937133789062, | |
| "learning_rate": 9.964516155915151e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 68 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1797.25, | |
| "epoch": 0.138, | |
| "grad_norm": 0.8277981877326965, | |
| "kl": 0.0012836456298828125, | |
| "learning_rate": 9.960469931131936e-07, | |
| "loss": 0.1147, | |
| "reward": 0.9375, | |
| "reward_std": 0.2651650309562683, | |
| "rewards/accuracy_reward": 0.75, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 69 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2000.25, | |
| "epoch": 0.14, | |
| "grad_norm": 0.8641435503959656, | |
| "kl": 0.0011048316955566406, | |
| "learning_rate": 9.956206309337066e-07, | |
| "loss": 0.0173, | |
| "reward": 0.25, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 70 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.142, | |
| "grad_norm": 0.009465116076171398, | |
| "kl": 0.00019884109497070312, | |
| "learning_rate": 9.951725498333448e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 71 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1618.25, | |
| "epoch": 0.144, | |
| "grad_norm": 0.8773428797721863, | |
| "kl": 0.0106658935546875, | |
| "learning_rate": 9.947027716509488e-07, | |
| "loss": 0.0216, | |
| "reward": 0.6875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 72 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.146, | |
| "grad_norm": 0.0040911422111094, | |
| "kl": 0.0006923675537109375, | |
| "learning_rate": 9.942113192828444e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 73 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.148, | |
| "grad_norm": 0.0059630973264575005, | |
| "kl": 0.000446319580078125, | |
| "learning_rate": 9.93698216681727e-07, | |
| "loss": 0.0, | |
| "reward": 0.75, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 74 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1836.75, | |
| "epoch": 0.15, | |
| "grad_norm": 1.146507978439331, | |
| "kl": 0.0004787445068359375, | |
| "learning_rate": 9.931634888554935e-07, | |
| "loss": -0.0918, | |
| "reward": 0.125, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 75 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.152, | |
| "grad_norm": 0.7911372184753418, | |
| "kl": 0.00030231475830078125, | |
| "learning_rate": 9.926071618660237e-07, | |
| "loss": 0.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 76 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.154, | |
| "grad_norm": 0.8156778216362, | |
| "kl": 0.0006389617919921875, | |
| "learning_rate": 9.9202926282791e-07, | |
| "loss": 0.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 77 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1915.25, | |
| "epoch": 0.156, | |
| "grad_norm": 0.005794746335595846, | |
| "kl": 0.0005588531494140625, | |
| "learning_rate": 9.91429819907136e-07, | |
| "loss": 0.0, | |
| "reward": 0.75, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 78 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1918.0, | |
| "epoch": 0.158, | |
| "grad_norm": 1.0309467315673828, | |
| "kl": 0.0005645751953125, | |
| "learning_rate": 9.908088623197048e-07, | |
| "loss": -0.0514, | |
| "reward": 0.125, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 79 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.16, | |
| "grad_norm": 0.9030879735946655, | |
| "kl": 0.0043182373046875, | |
| "learning_rate": 9.901664203302124e-07, | |
| "loss": 0.0002, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 80 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.162, | |
| "grad_norm": 0.8063321113586426, | |
| "kl": 0.00075531005859375, | |
| "learning_rate": 9.895025252503755e-07, | |
| "loss": 0.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 81 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.164, | |
| "grad_norm": 0.01648077741265297, | |
| "kl": 0.00039196014404296875, | |
| "learning_rate": 9.888172094375033e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 82 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1855.0, | |
| "epoch": 0.166, | |
| "grad_norm": 0.7015650272369385, | |
| "kl": 0.0013475418090820312, | |
| "learning_rate": 9.881105062929221e-07, | |
| "loss": 0.0001, | |
| "reward": 0.0625, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.0625, | |
| "step": 83 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.168, | |
| "grad_norm": 0.006011520978063345, | |
| "kl": 0.00021600723266601562, | |
| "learning_rate": 9.873824502603459e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 84 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1855.5, | |
| "epoch": 0.17, | |
| "grad_norm": 0.9796985983848572, | |
| "kl": 0.00054168701171875, | |
| "learning_rate": 9.866330768241983e-07, | |
| "loss": 0.0819, | |
| "reward": 0.5, | |
| "reward_std": 0.3535533845424652, | |
| "rewards/accuracy_reward": 0.25, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 85 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.172, | |
| "grad_norm": 0.006532273255288601, | |
| "kl": 0.0004496574401855469, | |
| "learning_rate": 9.85862422507884e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 86 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.174, | |
| "grad_norm": 0.007916338741779327, | |
| "kl": 0.0005373954772949219, | |
| "learning_rate": 9.850705248720068e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 87 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.176, | |
| "grad_norm": 0.006668766029179096, | |
| "kl": 0.00024175643920898438, | |
| "learning_rate": 9.8425742251254e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 88 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1533.5, | |
| "epoch": 0.178, | |
| "grad_norm": 0.01681283488869667, | |
| "kl": 0.0006351470947265625, | |
| "learning_rate": 9.83423155058946e-07, | |
| "loss": 0.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 89 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.18, | |
| "grad_norm": 0.6811097860336304, | |
| "kl": 0.0013413429260253906, | |
| "learning_rate": 9.825677631722435e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 90 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.182, | |
| "grad_norm": 0.7697494626045227, | |
| "kl": 0.0030584335327148438, | |
| "learning_rate": 9.816912885430258e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 91 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1913.25, | |
| "epoch": 0.184, | |
| "grad_norm": 1.1320937871932983, | |
| "kl": 0.001331329345703125, | |
| "learning_rate": 9.807937738894303e-07, | |
| "loss": 0.0536, | |
| "reward": 0.4375, | |
| "reward_std": 0.4419417306780815, | |
| "rewards/accuracy_reward": 0.25, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 92 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1742.75, | |
| "epoch": 0.186, | |
| "grad_norm": 0.6808719635009766, | |
| "kl": 0.0010509490966796875, | |
| "learning_rate": 9.798752629550546e-07, | |
| "loss": -0.1501, | |
| "reward": 0.3125, | |
| "reward_std": 0.2651650309562683, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.3125, | |
| "step": 93 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.188, | |
| "grad_norm": 0.007471662946045399, | |
| "kl": 0.0004634857177734375, | |
| "learning_rate": 9.78935800506826e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 94 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.19, | |
| "grad_norm": 0.010327517054975033, | |
| "kl": 0.000568389892578125, | |
| "learning_rate": 9.779754323328192e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 95 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.192, | |
| "grad_norm": 0.6696183681488037, | |
| "kl": 0.0009899139404296875, | |
| "learning_rate": 9.769942052400235e-07, | |
| "loss": 0.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 96 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.194, | |
| "grad_norm": 0.8246662020683289, | |
| "kl": 0.0017538070678710938, | |
| "learning_rate": 9.759921670520634e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 97 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1928.5, | |
| "epoch": 0.196, | |
| "grad_norm": 0.955489456653595, | |
| "kl": 0.0009098052978515625, | |
| "learning_rate": 9.749693666068663e-07, | |
| "loss": 0.0467, | |
| "reward": 0.375, | |
| "reward_std": 0.3535533770918846, | |
| "rewards/accuracy_reward": 0.25, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 98 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.198, | |
| "grad_norm": 0.03343038633465767, | |
| "kl": 0.0007419586181640625, | |
| "learning_rate": 9.739258537542835e-07, | |
| "loss": 0.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 99 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.2, | |
| "grad_norm": 0.006429341156035662, | |
| "kl": 0.001178741455078125, | |
| "learning_rate": 9.728616793536587e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 100 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.202, | |
| "grad_norm": 0.00873623974621296, | |
| "kl": 0.0005130767822265625, | |
| "learning_rate": 9.717768952713511e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 101 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1959.0, | |
| "epoch": 0.204, | |
| "grad_norm": 0.017306441441178322, | |
| "kl": 0.000946044921875, | |
| "learning_rate": 9.706715543782064e-07, | |
| "loss": 0.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 102 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.206, | |
| "grad_norm": 0.7247556447982788, | |
| "kl": 0.0008764266967773438, | |
| "learning_rate": 9.695457105469804e-07, | |
| "loss": 0.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 103 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.208, | |
| "grad_norm": 0.008871670812368393, | |
| "kl": 0.00033664703369140625, | |
| "learning_rate": 9.683994186497132e-07, | |
| "loss": 0.0, | |
| "reward": 0.75, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 104 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.21, | |
| "grad_norm": 0.007749219890683889, | |
| "kl": 0.00040531158447265625, | |
| "learning_rate": 9.672327345550543e-07, | |
| "loss": 0.0, | |
| "reward": 0.75, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 105 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.212, | |
| "grad_norm": 0.010708320885896683, | |
| "kl": 0.0010166168212890625, | |
| "learning_rate": 9.66045715125541e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 106 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.214, | |
| "grad_norm": 0.007295151706784964, | |
| "kl": 0.0003333091735839844, | |
| "learning_rate": 9.648384182148252e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 107 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.216, | |
| "grad_norm": 0.7922310829162598, | |
| "kl": 0.000408172607421875, | |
| "learning_rate": 9.636109026648554e-07, | |
| "loss": 0.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 108 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.218, | |
| "grad_norm": 0.007899758405983448, | |
| "kl": 0.0006399154663085938, | |
| "learning_rate": 9.623632283030077e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 109 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.22, | |
| "grad_norm": 0.9048980474472046, | |
| "kl": 0.001056671142578125, | |
| "learning_rate": 9.610954559391704e-07, | |
| "loss": 0.0, | |
| "reward": 0.6875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 110 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.222, | |
| "grad_norm": 0.010189698077738285, | |
| "kl": 0.00031948089599609375, | |
| "learning_rate": 9.598076473627796e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 111 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1794.5, | |
| "epoch": 0.224, | |
| "grad_norm": 1.0683528184890747, | |
| "kl": 0.0034933090209960938, | |
| "learning_rate": 9.58499865339809e-07, | |
| "loss": -0.1162, | |
| "reward": 0.6875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 112 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.226, | |
| "grad_norm": 0.011858138255774975, | |
| "kl": 0.000347137451171875, | |
| "learning_rate": 9.571721736097088e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 113 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.228, | |
| "grad_norm": 0.012356019578874111, | |
| "kl": 0.000885009765625, | |
| "learning_rate": 9.55824636882301e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 114 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1752.5, | |
| "epoch": 0.23, | |
| "grad_norm": 1.0531798601150513, | |
| "kl": 0.001102447509765625, | |
| "learning_rate": 9.54457320834625e-07, | |
| "loss": 0.1434, | |
| "reward": 0.1875, | |
| "reward_std": 0.2651650309562683, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 115 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.232, | |
| "grad_norm": 0.012715999968349934, | |
| "kl": 0.00096893310546875, | |
| "learning_rate": 9.530702921077358e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 116 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2033.5, | |
| "epoch": 0.234, | |
| "grad_norm": 0.7506431937217712, | |
| "kl": 0.0050945281982421875, | |
| "learning_rate": 9.516636183034564e-07, | |
| "loss": 0.0002, | |
| "reward": 0.0625, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.0625, | |
| "step": 117 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.236, | |
| "grad_norm": 0.020277904346585274, | |
| "kl": 0.00078582763671875, | |
| "learning_rate": 9.502373679810839e-07, | |
| "loss": 0.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 118 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1624.5, | |
| "epoch": 0.238, | |
| "grad_norm": 0.9856612086296082, | |
| "kl": 0.0022125244140625, | |
| "learning_rate": 9.487916106540465e-07, | |
| "loss": -0.0067, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 119 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.24, | |
| "grad_norm": 0.007632279768586159, | |
| "kl": 0.000583648681640625, | |
| "learning_rate": 9.473264167865171e-07, | |
| "loss": 0.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 120 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.242, | |
| "grad_norm": 0.007833893410861492, | |
| "kl": 0.0008401870727539062, | |
| "learning_rate": 9.458418577899774e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 121 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.244, | |
| "grad_norm": 0.00713867275044322, | |
| "kl": 0.0011892318725585938, | |
| "learning_rate": 9.443380060197385e-07, | |
| "loss": 0.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 122 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.246, | |
| "grad_norm": 0.011064039543271065, | |
| "kl": 0.0003185272216796875, | |
| "learning_rate": 9.428149347714143e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 123 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.248, | |
| "grad_norm": 0.009095462039113045, | |
| "kl": 0.0006237030029296875, | |
| "learning_rate": 9.412727182773486e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 124 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.25, | |
| "grad_norm": 0.007876625284552574, | |
| "kl": 0.00144195556640625, | |
| "learning_rate": 9.397114317029974e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 125 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.252, | |
| "grad_norm": 0.05358020216226578, | |
| "kl": 0.0010623931884765625, | |
| "learning_rate": 9.381311511432658e-07, | |
| "loss": 0.0, | |
| "reward": 0.625, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 126 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.254, | |
| "grad_norm": 0.030433854088187218, | |
| "kl": 0.0003848075866699219, | |
| "learning_rate": 9.36531953618799e-07, | |
| "loss": 0.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 127 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.256, | |
| "grad_norm": 0.852528989315033, | |
| "kl": 0.0011425018310546875, | |
| "learning_rate": 9.34913917072228e-07, | |
| "loss": 0.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 128 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.258, | |
| "grad_norm": 0.013770471327006817, | |
| "kl": 0.0007495880126953125, | |
| "learning_rate": 9.332771203643714e-07, | |
| "loss": 0.0, | |
| "reward": 0.75, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 129 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.26, | |
| "grad_norm": 0.7055062055587769, | |
| "kl": 0.0012664794921875, | |
| "learning_rate": 9.316216432703916e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 130 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.262, | |
| "grad_norm": 0.7149041295051575, | |
| "kl": 0.002803802490234375, | |
| "learning_rate": 9.299475664759068e-07, | |
| "loss": 0.0001, | |
| "reward": 0.0625, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.0625, | |
| "step": 131 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1480.25, | |
| "epoch": 0.264, | |
| "grad_norm": 0.014864159747958183, | |
| "kl": 0.031280517578125, | |
| "learning_rate": 9.282549715730579e-07, | |
| "loss": 0.0001, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 132 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.266, | |
| "grad_norm": 0.8071376085281372, | |
| "kl": 0.00067901611328125, | |
| "learning_rate": 9.265439410565328e-07, | |
| "loss": 0.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 133 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2047.25, | |
| "epoch": 0.268, | |
| "grad_norm": 0.009836402721703053, | |
| "kl": 0.0012359619140625, | |
| "learning_rate": 9.248145583195447e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 134 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2012.25, | |
| "epoch": 0.27, | |
| "grad_norm": 0.007896827533841133, | |
| "kl": 0.0009002685546875, | |
| "learning_rate": 9.230669076497687e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 135 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.272, | |
| "grad_norm": 0.0065347570925951, | |
| "kl": 0.000980377197265625, | |
| "learning_rate": 9.213010742252327e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 136 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.274, | |
| "grad_norm": 0.8659334778785706, | |
| "kl": 0.00147247314453125, | |
| "learning_rate": 9.195171441101668e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 137 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.276, | |
| "grad_norm": 0.009184672497212887, | |
| "kl": 0.001373291015625, | |
| "learning_rate": 9.177152042508077e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 138 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.278, | |
| "grad_norm": 0.00881196279078722, | |
| "kl": 0.001476287841796875, | |
| "learning_rate": 9.158953424711624e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 139 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1876.0, | |
| "epoch": 0.28, | |
| "grad_norm": 0.7441470623016357, | |
| "kl": 0.0006885528564453125, | |
| "learning_rate": 9.140576474687263e-07, | |
| "loss": 0.06, | |
| "reward": 0.3125, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.3125, | |
| "step": 140 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.282, | |
| "grad_norm": 0.008356408216059208, | |
| "kl": 0.00101470947265625, | |
| "learning_rate": 9.122022088101613e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 141 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1991.5, | |
| "epoch": 0.284, | |
| "grad_norm": 0.9580811262130737, | |
| "kl": 0.006805419921875, | |
| "learning_rate": 9.103291169269299e-07, | |
| "loss": 0.0003, | |
| "reward": 0.5625, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.0625, | |
| "step": 142 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.286, | |
| "grad_norm": 0.008105803281068802, | |
| "kl": 0.002086639404296875, | |
| "learning_rate": 9.084384631108882e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 143 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.288, | |
| "grad_norm": 0.007048532832413912, | |
| "kl": 0.001430511474609375, | |
| "learning_rate": 9.065303395098358e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 144 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1714.5, | |
| "epoch": 0.29, | |
| "grad_norm": 0.881592869758606, | |
| "kl": 0.001071929931640625, | |
| "learning_rate": 9.046048391230247e-07, | |
| "loss": 0.1707, | |
| "reward": 0.25, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 145 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.292, | |
| "grad_norm": 0.008095495402812958, | |
| "kl": 0.00165557861328125, | |
| "learning_rate": 9.026620557966279e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 146 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.294, | |
| "grad_norm": 0.019286898896098137, | |
| "kl": 0.00104522705078125, | |
| "learning_rate": 9.007020842191634e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 147 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.296, | |
| "grad_norm": 0.007973677478730679, | |
| "kl": 0.0015087127685546875, | |
| "learning_rate": 8.987250199168808e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 148 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.298, | |
| "grad_norm": 0.007949120365083218, | |
| "kl": 0.0004978179931640625, | |
| "learning_rate": 8.967309592491052e-07, | |
| "loss": 0.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 149 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.3, | |
| "grad_norm": 0.007726718205958605, | |
| "kl": 0.001689910888671875, | |
| "learning_rate": 8.9471999940354e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 150 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.302, | |
| "grad_norm": 0.00826709158718586, | |
| "kl": 0.0013580322265625, | |
| "learning_rate": 8.926922383915315e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 151 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.304, | |
| "grad_norm": 0.007963276468217373, | |
| "kl": 0.00203704833984375, | |
| "learning_rate": 8.906477750432903e-07, | |
| "loss": 0.0001, | |
| "reward": 0.75, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 152 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.306, | |
| "grad_norm": 0.008207273669540882, | |
| "kl": 0.0008563995361328125, | |
| "learning_rate": 8.88586709003076e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 153 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.308, | |
| "grad_norm": 0.010204545222222805, | |
| "kl": 0.0006160736083984375, | |
| "learning_rate": 8.865091407243394e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 154 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.31, | |
| "grad_norm": 0.00880539882928133, | |
| "kl": 0.001453399658203125, | |
| "learning_rate": 8.844151714648274e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 155 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.312, | |
| "grad_norm": 0.010449129156768322, | |
| "kl": 0.0010318756103515625, | |
| "learning_rate": 8.823049032816478e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 156 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.314, | |
| "grad_norm": 0.010188284330070019, | |
| "kl": 0.000789642333984375, | |
| "learning_rate": 8.801784390262943e-07, | |
| "loss": 0.0, | |
| "reward": 0.75, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 157 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1963.5, | |
| "epoch": 0.316, | |
| "grad_norm": 0.7694103717803955, | |
| "kl": 0.000640869140625, | |
| "learning_rate": 8.780358823396352e-07, | |
| "loss": 0.0318, | |
| "reward": 0.0625, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.0625, | |
| "step": 158 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1807.0, | |
| "epoch": 0.318, | |
| "grad_norm": 0.8925904631614685, | |
| "kl": 0.001445770263671875, | |
| "learning_rate": 8.758773376468604e-07, | |
| "loss": -0.1087, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 159 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.32, | |
| "grad_norm": 0.7490972280502319, | |
| "kl": 0.01116180419921875, | |
| "learning_rate": 8.737029101523929e-07, | |
| "loss": 0.0004, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 160 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.322, | |
| "grad_norm": 0.008408155292272568, | |
| "kl": 0.0014629364013671875, | |
| "learning_rate": 8.715127058347614e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 161 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.324, | |
| "grad_norm": 0.8539191484451294, | |
| "kl": 0.0008831024169921875, | |
| "learning_rate": 8.693068314414344e-07, | |
| "loss": 0.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 162 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.326, | |
| "grad_norm": 0.009831397794187069, | |
| "kl": 0.000682830810546875, | |
| "learning_rate": 8.670853944836176e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 163 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.328, | |
| "grad_norm": 0.011773304082453251, | |
| "kl": 0.001129150390625, | |
| "learning_rate": 8.648485032310144e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 164 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.33, | |
| "grad_norm": 0.7276328206062317, | |
| "kl": 0.001377105712890625, | |
| "learning_rate": 8.625962667065487e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 165 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1512.75, | |
| "epoch": 0.332, | |
| "grad_norm": 0.008593901991844177, | |
| "kl": 0.0006933212280273438, | |
| "learning_rate": 8.603287946810513e-07, | |
| "loss": 0.0, | |
| "reward": 0.375, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.375, | |
| "step": 166 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.334, | |
| "grad_norm": 0.011681273579597473, | |
| "kl": 0.000759124755859375, | |
| "learning_rate": 8.580461976679099e-07, | |
| "loss": 0.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 167 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.336, | |
| "grad_norm": 0.9719880819320679, | |
| "kl": 0.00112152099609375, | |
| "learning_rate": 8.557485869176825e-07, | |
| "loss": 0.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 168 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1850.75, | |
| "epoch": 0.338, | |
| "grad_norm": 0.017566794529557228, | |
| "kl": 0.002292633056640625, | |
| "learning_rate": 8.534360744126753e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 169 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2016.0, | |
| "epoch": 0.34, | |
| "grad_norm": 1.1285712718963623, | |
| "kl": 0.00118255615234375, | |
| "learning_rate": 8.511087728614862e-07, | |
| "loss": 0.0114, | |
| "reward": 0.3125, | |
| "reward_std": 0.2651650384068489, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.3125, | |
| "step": 170 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.342, | |
| "grad_norm": 0.009710059501230717, | |
| "kl": 0.001407623291015625, | |
| "learning_rate": 8.487667956935087e-07, | |
| "loss": 0.0001, | |
| "reward": 0.75, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 171 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.344, | |
| "grad_norm": 0.009776381775736809, | |
| "kl": 0.0014629364013671875, | |
| "learning_rate": 8.464102570534061e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 172 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.346, | |
| "grad_norm": 0.00871388241648674, | |
| "kl": 0.000507354736328125, | |
| "learning_rate": 8.440392717955475e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 173 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.348, | |
| "grad_norm": 0.00912429578602314, | |
| "kl": 0.000762939453125, | |
| "learning_rate": 8.416539554784089e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 174 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.35, | |
| "grad_norm": 0.011816666461527348, | |
| "kl": 0.0008449554443359375, | |
| "learning_rate": 8.392544243589427e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 175 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.352, | |
| "grad_norm": 0.00985003262758255, | |
| "kl": 0.001529693603515625, | |
| "learning_rate": 8.368407953869103e-07, | |
| "loss": 0.0001, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 176 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.354, | |
| "grad_norm": 0.00919476430863142, | |
| "kl": 0.00167083740234375, | |
| "learning_rate": 8.344131861991828e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 177 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.356, | |
| "grad_norm": 0.01162977609783411, | |
| "kl": 0.00091552734375, | |
| "learning_rate": 8.319717151140072e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 178 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1970.5, | |
| "epoch": 0.358, | |
| "grad_norm": 0.01332745049148798, | |
| "kl": 0.001888275146484375, | |
| "learning_rate": 8.295165011252396e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 179 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1730.75, | |
| "epoch": 0.36, | |
| "grad_norm": 0.013342260383069515, | |
| "kl": 0.00089263916015625, | |
| "learning_rate": 8.270476638965461e-07, | |
| "loss": 0.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 180 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.362, | |
| "grad_norm": 0.012930563651025295, | |
| "kl": 0.001483917236328125, | |
| "learning_rate": 8.245653237555705e-07, | |
| "loss": 0.0001, | |
| "reward": 0.625, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 181 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.364, | |
| "grad_norm": 0.03247583284974098, | |
| "kl": 0.00113677978515625, | |
| "learning_rate": 8.220696016880687e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 182 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.366, | |
| "grad_norm": 0.8804360032081604, | |
| "kl": 0.0019130706787109375, | |
| "learning_rate": 8.195606193320136e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 183 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.368, | |
| "grad_norm": 0.007631482556462288, | |
| "kl": 0.001468658447265625, | |
| "learning_rate": 8.170384989716657e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 184 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.37, | |
| "grad_norm": 0.008286840282380581, | |
| "kl": 0.001613616943359375, | |
| "learning_rate": 8.145033635316128e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 185 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.372, | |
| "grad_norm": 0.016586236655712128, | |
| "kl": 0.0007076263427734375, | |
| "learning_rate": 8.119553365707802e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 186 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.374, | |
| "grad_norm": 0.010034182108938694, | |
| "kl": 0.0008754730224609375, | |
| "learning_rate": 8.093945422764069e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 187 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.376, | |
| "grad_norm": 0.7020014524459839, | |
| "kl": 0.0012836456298828125, | |
| "learning_rate": 8.068211054579943e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 188 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.378, | |
| "grad_norm": 1.163500189781189, | |
| "kl": 0.0016021728515625, | |
| "learning_rate": 8.04235151541222e-07, | |
| "loss": 0.0001, | |
| "reward": 0.125, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 189 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.38, | |
| "grad_norm": 0.8533800840377808, | |
| "kl": 0.0010223388671875, | |
| "learning_rate": 8.01636806561836e-07, | |
| "loss": 0.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 190 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.382, | |
| "grad_norm": 0.8227788805961609, | |
| "kl": 0.0030975341796875, | |
| "learning_rate": 7.990261971595048e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 191 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.384, | |
| "grad_norm": 0.011001263745129108, | |
| "kl": 0.0008344650268554688, | |
| "learning_rate": 7.964034505716476e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 192 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.386, | |
| "grad_norm": 0.009392702020704746, | |
| "kl": 0.001972198486328125, | |
| "learning_rate": 7.93768694627233e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 193 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1788.25, | |
| "epoch": 0.388, | |
| "grad_norm": 1.1597681045532227, | |
| "kl": 0.01373291015625, | |
| "learning_rate": 7.911220577405484e-07, | |
| "loss": 0.1207, | |
| "reward": 0.1875, | |
| "reward_std": 0.2651650384068489, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 194 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.39, | |
| "grad_norm": 0.00962373148649931, | |
| "kl": 0.0009326934814453125, | |
| "learning_rate": 7.884636689049422e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 195 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2025.0, | |
| "epoch": 0.392, | |
| "grad_norm": 0.009538036771118641, | |
| "kl": 0.00116729736328125, | |
| "learning_rate": 7.857936576865356e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 196 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.394, | |
| "grad_norm": 0.01050383411347866, | |
| "kl": 0.0007953643798828125, | |
| "learning_rate": 7.831121542179086e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 197 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.396, | |
| "grad_norm": 0.009053800255060196, | |
| "kl": 0.001438140869140625, | |
| "learning_rate": 7.804192891917571e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 198 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.398, | |
| "grad_norm": 0.00897100381553173, | |
| "kl": 0.001312255859375, | |
| "learning_rate": 7.777151938545235e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 199 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.4, | |
| "grad_norm": 0.01025310903787613, | |
| "kl": 0.001468658447265625, | |
| "learning_rate": 7.75e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 200 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.402, | |
| "grad_norm": 0.054522059857845306, | |
| "kl": 0.0010223388671875, | |
| "learning_rate": 7.72273839962904e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 201 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1953.5, | |
| "epoch": 0.404, | |
| "grad_norm": 0.037722665816545486, | |
| "kl": 0.0011157989501953125, | |
| "learning_rate": 7.695368466124296e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 202 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.406, | |
| "grad_norm": 0.008431609719991684, | |
| "kl": 0.0020599365234375, | |
| "learning_rate": 7.667891533457718e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 203 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1929.0, | |
| "epoch": 0.408, | |
| "grad_norm": 0.9533175826072693, | |
| "kl": 0.001682281494140625, | |
| "learning_rate": 7.640308940816239e-07, | |
| "loss": 0.0001, | |
| "reward": 0.5625, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.0625, | |
| "step": 204 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2036.0, | |
| "epoch": 0.41, | |
| "grad_norm": 0.013962327502667904, | |
| "kl": 0.0011119842529296875, | |
| "learning_rate": 7.612622032536507e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 205 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.412, | |
| "grad_norm": 0.010255957953631878, | |
| "kl": 0.0009479522705078125, | |
| "learning_rate": 7.584832158039378e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 206 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.414, | |
| "grad_norm": 0.01276308298110962, | |
| "kl": 0.0014495849609375, | |
| "learning_rate": 7.556940671764124e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 207 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1809.5, | |
| "epoch": 0.416, | |
| "grad_norm": 0.01625184714794159, | |
| "kl": 0.00093841552734375, | |
| "learning_rate": 7.528948933102438e-07, | |
| "loss": 0.0, | |
| "reward": 0.75, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 208 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.418, | |
| "grad_norm": 0.010438801720738411, | |
| "kl": 0.0012836456298828125, | |
| "learning_rate": 7.500858306332172e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 209 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.42, | |
| "grad_norm": 0.011556737124919891, | |
| "kl": 0.00215911865234375, | |
| "learning_rate": 7.472670160550848e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 210 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.422, | |
| "grad_norm": 0.009882017970085144, | |
| "kl": 0.0005950927734375, | |
| "learning_rate": 7.444385869608921e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 211 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1786.0, | |
| "epoch": 0.424, | |
| "grad_norm": 0.8873146176338196, | |
| "kl": 0.002838134765625, | |
| "learning_rate": 7.416006812042827e-07, | |
| "loss": -0.0358, | |
| "reward": 0.375, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.375, | |
| "step": 212 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.426, | |
| "grad_norm": 0.01222989521920681, | |
| "kl": 0.0014190673828125, | |
| "learning_rate": 7.387534371007797e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 213 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.428, | |
| "grad_norm": 0.009303831495344639, | |
| "kl": 0.00127410888671875, | |
| "learning_rate": 7.358969934210438e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 214 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.43, | |
| "grad_norm": 0.0105022257193923, | |
| "kl": 0.0008087158203125, | |
| "learning_rate": 7.330314893841101e-07, | |
| "loss": 0.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 215 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.432, | |
| "grad_norm": 0.763167679309845, | |
| "kl": 0.001544952392578125, | |
| "learning_rate": 7.301570646506027e-07, | |
| "loss": 0.0001, | |
| "reward": 0.3125, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.3125, | |
| "step": 216 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.434, | |
| "grad_norm": 0.010868730954825878, | |
| "kl": 0.001415252685546875, | |
| "learning_rate": 7.27273859315928e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 217 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.436, | |
| "grad_norm": 0.016456812620162964, | |
| "kl": 0.0006694793701171875, | |
| "learning_rate": 7.243820139034464e-07, | |
| "loss": 0.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 218 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1947.75, | |
| "epoch": 0.438, | |
| "grad_norm": 0.5713726282119751, | |
| "kl": 0.0039825439453125, | |
| "learning_rate": 7.214816693576234e-07, | |
| "loss": 0.0002, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 219 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.44, | |
| "grad_norm": 0.008825725875794888, | |
| "kl": 0.000873565673828125, | |
| "learning_rate": 7.185729670371604e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 220 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.442, | |
| "grad_norm": 0.015775861218571663, | |
| "kl": 0.0016326904296875, | |
| "learning_rate": 7.156560487081051e-07, | |
| "loss": 0.0001, | |
| "reward": 0.75, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 221 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1892.75, | |
| "epoch": 0.444, | |
| "grad_norm": 0.011794094927608967, | |
| "kl": 0.002288818359375, | |
| "learning_rate": 7.127310565369415e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 222 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.446, | |
| "grad_norm": 0.8902475833892822, | |
| "kl": 0.0021953582763671875, | |
| "learning_rate": 7.097981330836616e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 223 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1976.5, | |
| "epoch": 0.448, | |
| "grad_norm": 0.01060924306511879, | |
| "kl": 0.00159454345703125, | |
| "learning_rate": 7.068574212948169e-07, | |
| "loss": 0.0001, | |
| "reward": 0.75, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 224 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1884.0, | |
| "epoch": 0.45, | |
| "grad_norm": 0.7874477505683899, | |
| "kl": 0.00165557861328125, | |
| "learning_rate": 7.039090644965509e-07, | |
| "loss": 0.0675, | |
| "reward": 0.375, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.375, | |
| "step": 225 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.452, | |
| "grad_norm": 0.009480384178459644, | |
| "kl": 0.001316070556640625, | |
| "learning_rate": 7.009532063876148e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 226 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.454, | |
| "grad_norm": 0.8971606492996216, | |
| "kl": 0.00217437744140625, | |
| "learning_rate": 6.979899910323624e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 227 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1950.75, | |
| "epoch": 0.456, | |
| "grad_norm": 0.013516876846551895, | |
| "kl": 0.001598358154296875, | |
| "learning_rate": 6.950195628537299e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 228 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.458, | |
| "grad_norm": 0.012519976124167442, | |
| "kl": 0.0012359619140625, | |
| "learning_rate": 6.920420666261961e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 229 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1604.75, | |
| "epoch": 0.46, | |
| "grad_norm": 1.0533803701400757, | |
| "kl": 0.0024871826171875, | |
| "learning_rate": 6.890576474687263e-07, | |
| "loss": 0.1645, | |
| "reward": 0.75, | |
| "reward_std": 0.7071067541837692, | |
| "rewards/accuracy_reward": 0.25, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.5, | |
| "step": 230 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.462, | |
| "grad_norm": 0.04883728548884392, | |
| "kl": 0.0012359619140625, | |
| "learning_rate": 6.860664508377001e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 231 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.464, | |
| "grad_norm": 0.01617475040256977, | |
| "kl": 0.00106048583984375, | |
| "learning_rate": 6.83068622519821e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 232 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.466, | |
| "grad_norm": 0.01098883431404829, | |
| "kl": 0.00160980224609375, | |
| "learning_rate": 6.800643086250121e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 233 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.468, | |
| "grad_norm": 0.010283930227160454, | |
| "kl": 0.001750946044921875, | |
| "learning_rate": 6.770536555792944e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 234 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.47, | |
| "grad_norm": 0.7373048663139343, | |
| "kl": 0.00131988525390625, | |
| "learning_rate": 6.740368101176495e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 235 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.472, | |
| "grad_norm": 0.01158232893794775, | |
| "kl": 0.001461029052734375, | |
| "learning_rate": 6.710139192768694e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 236 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.474, | |
| "grad_norm": 0.8555125594139099, | |
| "kl": 0.001068115234375, | |
| "learning_rate": 6.679851303883891e-07, | |
| "loss": 0.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 237 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.476, | |
| "grad_norm": 0.8464781641960144, | |
| "kl": 0.006557464599609375, | |
| "learning_rate": 6.649505910711058e-07, | |
| "loss": 0.0003, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 238 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.478, | |
| "grad_norm": 0.14680787920951843, | |
| "kl": 0.003086090087890625, | |
| "learning_rate": 6.619104492241847e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 239 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.48, | |
| "grad_norm": 0.8916466236114502, | |
| "kl": 0.010036468505859375, | |
| "learning_rate": 6.588648530198504e-07, | |
| "loss": 0.0004, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 240 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.482, | |
| "grad_norm": 0.012613446451723576, | |
| "kl": 0.001827239990234375, | |
| "learning_rate": 6.558139508961654e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 241 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.484, | |
| "grad_norm": 0.012667362578213215, | |
| "kl": 0.001438140869140625, | |
| "learning_rate": 6.527578915497951e-07, | |
| "loss": 0.0001, | |
| "reward": 0.75, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 242 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.486, | |
| "grad_norm": 0.011633522808551788, | |
| "kl": 0.0007953643798828125, | |
| "learning_rate": 6.496968239287603e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 243 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.488, | |
| "grad_norm": 0.010322234593331814, | |
| "kl": 0.00118255615234375, | |
| "learning_rate": 6.466308972251785e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 244 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.49, | |
| "grad_norm": 0.013764915056526661, | |
| "kl": 0.001338958740234375, | |
| "learning_rate": 6.435602608679916e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 245 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.492, | |
| "grad_norm": 0.016423719003796577, | |
| "kl": 0.0011043548583984375, | |
| "learning_rate": 6.404850645156841e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 246 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.494, | |
| "grad_norm": 0.011886836029589176, | |
| "kl": 0.0008907318115234375, | |
| "learning_rate": 6.374054580489873e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 247 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.496, | |
| "grad_norm": 0.013753964565694332, | |
| "kl": 0.0009822845458984375, | |
| "learning_rate": 6.343215915635761e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 248 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.498, | |
| "grad_norm": 0.012257426045835018, | |
| "kl": 0.00170135498046875, | |
| "learning_rate": 6.31233615362752e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 249 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.5, | |
| "grad_norm": 0.7262430191040039, | |
| "kl": 0.00194549560546875, | |
| "learning_rate": 6.281416799501187e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 250 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.502, | |
| "grad_norm": 0.7270606756210327, | |
| "kl": 0.0007076263427734375, | |
| "learning_rate": 6.25045936022246e-07, | |
| "loss": 0.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 251 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.504, | |
| "grad_norm": 0.017829036340117455, | |
| "kl": 0.0005779266357421875, | |
| "learning_rate": 6.219465344613258e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 252 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.506, | |
| "grad_norm": 0.0317312628030777, | |
| "kl": 0.0013294219970703125, | |
| "learning_rate": 6.188436263278172e-07, | |
| "loss": 0.0001, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 253 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1972.25, | |
| "epoch": 0.508, | |
| "grad_norm": 0.026640823110938072, | |
| "kl": 0.000835418701171875, | |
| "learning_rate": 6.157373628530852e-07, | |
| "loss": 0.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 254 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.51, | |
| "grad_norm": 0.012969509698450565, | |
| "kl": 0.001567840576171875, | |
| "learning_rate": 6.126278954320294e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 255 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.512, | |
| "grad_norm": 0.012548292055726051, | |
| "kl": 0.0010986328125, | |
| "learning_rate": 6.095153756157051e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 256 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.514, | |
| "grad_norm": 0.008928977884352207, | |
| "kl": 0.000873565673828125, | |
| "learning_rate": 6.06399955103937e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 257 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.516, | |
| "grad_norm": 0.8101487755775452, | |
| "kl": 0.0009307861328125, | |
| "learning_rate": 6.032817857379256e-07, | |
| "loss": 0.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 258 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.518, | |
| "grad_norm": 0.8978201746940613, | |
| "kl": 0.0007915496826171875, | |
| "learning_rate": 6.001610194928464e-07, | |
| "loss": 0.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 259 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.52, | |
| "grad_norm": 1.092624306678772, | |
| "kl": 0.00101470947265625, | |
| "learning_rate": 5.97037808470444e-07, | |
| "loss": 0.0, | |
| "reward": 0.125, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 260 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.522, | |
| "grad_norm": 0.7928449511528015, | |
| "kl": 0.0011920928955078125, | |
| "learning_rate": 5.939123048916173e-07, | |
| "loss": 0.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 261 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1534.75, | |
| "epoch": 0.524, | |
| "grad_norm": 1.0032625198364258, | |
| "kl": 0.0021257400512695312, | |
| "learning_rate": 5.907846610890011e-07, | |
| "loss": 0.0615, | |
| "reward": 0.375, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.375, | |
| "step": 262 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.526, | |
| "grad_norm": 0.011720544658601284, | |
| "kl": 0.000644683837890625, | |
| "learning_rate": 5.87655029499542e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 263 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.528, | |
| "grad_norm": 0.011771513149142265, | |
| "kl": 0.0007781982421875, | |
| "learning_rate": 5.845235626570683e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 264 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.53, | |
| "grad_norm": 0.013503954745829105, | |
| "kl": 0.001155853271484375, | |
| "learning_rate": 5.813904131848564e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 265 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.532, | |
| "grad_norm": 0.09234623610973358, | |
| "kl": 0.0018482208251953125, | |
| "learning_rate": 5.78255733788191e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 266 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.534, | |
| "grad_norm": 0.011625738814473152, | |
| "kl": 0.0007114410400390625, | |
| "learning_rate": 5.751196772469237e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 267 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1589.75, | |
| "epoch": 0.536, | |
| "grad_norm": 0.9924006462097168, | |
| "kl": 0.0024566650390625, | |
| "learning_rate": 5.71982396408026e-07, | |
| "loss": -0.0413, | |
| "reward": 0.875, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.375, | |
| "step": 268 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.538, | |
| "grad_norm": 0.726823627948761, | |
| "kl": 0.001861572265625, | |
| "learning_rate": 5.688440441781398e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 269 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.54, | |
| "grad_norm": 0.011368845589458942, | |
| "kl": 0.001186370849609375, | |
| "learning_rate": 5.657047735161255e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 270 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.542, | |
| "grad_norm": 0.014150974340736866, | |
| "kl": 0.0012617111206054688, | |
| "learning_rate": 5.625647374256061e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 271 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.544, | |
| "grad_norm": 0.03309360519051552, | |
| "kl": 0.00139617919921875, | |
| "learning_rate": 5.594240889475106e-07, | |
| "loss": 0.0001, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 272 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.546, | |
| "grad_norm": 0.027406711131334305, | |
| "kl": 0.002048492431640625, | |
| "learning_rate": 5.562829811526154e-07, | |
| "loss": 0.0001, | |
| "reward": 0.75, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 273 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1862.75, | |
| "epoch": 0.548, | |
| "grad_norm": 0.872232973575592, | |
| "kl": 0.001857757568359375, | |
| "learning_rate": 5.531415671340826e-07, | |
| "loss": 0.0781, | |
| "reward": 0.375, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.375, | |
| "step": 274 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1893.5, | |
| "epoch": 0.55, | |
| "grad_norm": 0.013754754327237606, | |
| "kl": 0.00106048583984375, | |
| "learning_rate": 5.5e-07, | |
| "loss": 0.0, | |
| "reward": 0.75, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 275 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.552, | |
| "grad_norm": 0.8218861222267151, | |
| "kl": 0.0015106201171875, | |
| "learning_rate": 5.468584328659172e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 276 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.554, | |
| "grad_norm": 0.01286914199590683, | |
| "kl": 0.001323699951171875, | |
| "learning_rate": 5.437170188473847e-07, | |
| "loss": 0.0001, | |
| "reward": 0.75, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 277 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.556, | |
| "grad_norm": 0.8357925415039062, | |
| "kl": 0.00146484375, | |
| "learning_rate": 5.405759110524894e-07, | |
| "loss": 0.0001, | |
| "reward": 0.6875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 278 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.558, | |
| "grad_norm": 0.011115595698356628, | |
| "kl": 0.001163482666015625, | |
| "learning_rate": 5.37435262574394e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 279 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.56, | |
| "grad_norm": 0.7368820309638977, | |
| "kl": 0.0016918182373046875, | |
| "learning_rate": 5.342952264838747e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 280 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1845.5, | |
| "epoch": 0.562, | |
| "grad_norm": 0.8080930709838867, | |
| "kl": 0.002285003662109375, | |
| "learning_rate": 5.311559558218603e-07, | |
| "loss": 0.0872, | |
| "reward": 0.375, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.375, | |
| "step": 281 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.564, | |
| "grad_norm": 0.6212303638458252, | |
| "kl": 0.0017833709716796875, | |
| "learning_rate": 5.28017603591974e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 282 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.566, | |
| "grad_norm": 0.8698393106460571, | |
| "kl": 0.001556396484375, | |
| "learning_rate": 5.248803227530763e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 283 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.568, | |
| "grad_norm": 0.6195830702781677, | |
| "kl": 0.001125335693359375, | |
| "learning_rate": 5.21744266211809e-07, | |
| "loss": 0.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 284 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.57, | |
| "grad_norm": 0.014679288491606712, | |
| "kl": 0.00112152099609375, | |
| "learning_rate": 5.186095868151436e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 285 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.572, | |
| "grad_norm": 0.011439023539423943, | |
| "kl": 0.000751495361328125, | |
| "learning_rate": 5.154764373429315e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 286 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.574, | |
| "grad_norm": 0.013943897560238838, | |
| "kl": 0.00086212158203125, | |
| "learning_rate": 5.123449705004581e-07, | |
| "loss": 0.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 287 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.576, | |
| "grad_norm": 0.011966060847043991, | |
| "kl": 0.001087188720703125, | |
| "learning_rate": 5.09215338910999e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 288 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.578, | |
| "grad_norm": 0.7932072877883911, | |
| "kl": 0.002170562744140625, | |
| "learning_rate": 5.060876951083828e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 289 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.58, | |
| "grad_norm": 0.01169038936495781, | |
| "kl": 0.001129150390625, | |
| "learning_rate": 5.02962191529556e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 290 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.582, | |
| "grad_norm": 0.85643470287323, | |
| "kl": 0.002471923828125, | |
| "learning_rate": 4.998389805071536e-07, | |
| "loss": 0.0001, | |
| "reward": 0.125, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 291 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.584, | |
| "grad_norm": 1.001603364944458, | |
| "kl": 0.01709747314453125, | |
| "learning_rate": 4.967182142620745e-07, | |
| "loss": 0.0007, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 292 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1886.25, | |
| "epoch": 0.586, | |
| "grad_norm": 0.7774127721786499, | |
| "kl": 0.001667022705078125, | |
| "learning_rate": 4.93600044896063e-07, | |
| "loss": 0.0664, | |
| "reward": 0.375, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.375, | |
| "step": 293 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.588, | |
| "grad_norm": 1.0563451051712036, | |
| "kl": 0.00324249267578125, | |
| "learning_rate": 4.904846243842949e-07, | |
| "loss": 0.0001, | |
| "reward": 0.125, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 294 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.59, | |
| "grad_norm": 0.012082475237548351, | |
| "kl": 0.00081634521484375, | |
| "learning_rate": 4.873721045679706e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 295 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2003.5, | |
| "epoch": 0.592, | |
| "grad_norm": 0.590258002281189, | |
| "kl": 0.001506805419921875, | |
| "learning_rate": 4.842626371469149e-07, | |
| "loss": 0.0161, | |
| "reward": 0.625, | |
| "reward_std": 0.5303300619125366, | |
| "rewards/accuracy_reward": 0.25, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.375, | |
| "step": 296 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.594, | |
| "grad_norm": 0.7182537913322449, | |
| "kl": 0.00550079345703125, | |
| "learning_rate": 4.811563736721829e-07, | |
| "loss": 0.0002, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 297 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1947.25, | |
| "epoch": 0.596, | |
| "grad_norm": 1.0634351968765259, | |
| "kl": 0.0025787353515625, | |
| "learning_rate": 4.780534655386743e-07, | |
| "loss": 0.0387, | |
| "reward": 0.6875, | |
| "reward_std": 0.6187184080481529, | |
| "rewards/accuracy_reward": 0.25, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4375, | |
| "step": 298 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.598, | |
| "grad_norm": 0.054920367896556854, | |
| "kl": 0.00203704833984375, | |
| "learning_rate": 4.749540639777539e-07, | |
| "loss": 0.0001, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 299 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.6, | |
| "grad_norm": 0.012694926001131535, | |
| "kl": 0.000698089599609375, | |
| "learning_rate": 4.7185832004988133e-07, | |
| "loss": 0.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 300 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1795.5, | |
| "epoch": 0.602, | |
| "grad_norm": 0.759120523929596, | |
| "kl": 0.0024261474609375, | |
| "learning_rate": 4.68766384637248e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 301 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.604, | |
| "grad_norm": 0.010465751402080059, | |
| "kl": 0.001453399658203125, | |
| "learning_rate": 4.656784084364238e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 302 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1853.0, | |
| "epoch": 0.606, | |
| "grad_norm": 0.8468216061592102, | |
| "kl": 0.00121307373046875, | |
| "learning_rate": 4.6259454195101267e-07, | |
| "loss": -0.0831, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 303 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.608, | |
| "grad_norm": 0.9815247654914856, | |
| "kl": 0.001689910888671875, | |
| "learning_rate": 4.59514935484316e-07, | |
| "loss": 0.0001, | |
| "reward": 0.125, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 304 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1868.75, | |
| "epoch": 0.61, | |
| "grad_norm": 0.8037129044532776, | |
| "kl": 0.000972747802734375, | |
| "learning_rate": 4.5643973913200837e-07, | |
| "loss": 0.075, | |
| "reward": 0.375, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.375, | |
| "step": 305 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.612, | |
| "grad_norm": 0.011851584538817406, | |
| "kl": 0.001220703125, | |
| "learning_rate": 4.5336910277482155e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 306 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.614, | |
| "grad_norm": 0.012452667579054832, | |
| "kl": 0.000675201416015625, | |
| "learning_rate": 4.503031760712397e-07, | |
| "loss": 0.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 307 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.616, | |
| "grad_norm": 0.8061856627464294, | |
| "kl": 0.001941680908203125, | |
| "learning_rate": 4.4724210845020494e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 308 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.618, | |
| "grad_norm": 0.011598404496908188, | |
| "kl": 0.0009021759033203125, | |
| "learning_rate": 4.441860491038345e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 309 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.62, | |
| "grad_norm": 0.013049165718257427, | |
| "kl": 0.0008697509765625, | |
| "learning_rate": 4.4113514698014953e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 310 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.622, | |
| "grad_norm": 0.01000931765884161, | |
| "kl": 0.0010738372802734375, | |
| "learning_rate": 4.3808955077581546e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 311 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.624, | |
| "grad_norm": 0.017103025689721107, | |
| "kl": 0.00090789794921875, | |
| "learning_rate": 4.350494089288943e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 312 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.626, | |
| "grad_norm": 0.015600494109094143, | |
| "kl": 0.001026153564453125, | |
| "learning_rate": 4.3201486961161093e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 313 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.628, | |
| "grad_norm": 0.68843674659729, | |
| "kl": 0.0009326934814453125, | |
| "learning_rate": 4.2898608072313045e-07, | |
| "loss": 0.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 314 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.63, | |
| "grad_norm": 0.02028859592974186, | |
| "kl": 0.001399993896484375, | |
| "learning_rate": 4.2596318988235037e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 315 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.632, | |
| "grad_norm": 0.010004539042711258, | |
| "kl": 0.0006055831909179688, | |
| "learning_rate": 4.2294634442070553e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 316 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.634, | |
| "grad_norm": 0.012407040223479271, | |
| "kl": 0.001323699951171875, | |
| "learning_rate": 4.1993569137498776e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 317 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.636, | |
| "grad_norm": 0.7615489363670349, | |
| "kl": 0.001617431640625, | |
| "learning_rate": 4.1693137748017915e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 318 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.638, | |
| "grad_norm": 0.7873152494430542, | |
| "kl": 0.00064849853515625, | |
| "learning_rate": 4.1393354916230005e-07, | |
| "loss": 0.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 319 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.64, | |
| "grad_norm": 0.012965940870344639, | |
| "kl": 0.000850677490234375, | |
| "learning_rate": 4.1094235253127374e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 320 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.642, | |
| "grad_norm": 0.013525367714464664, | |
| "kl": 0.000804901123046875, | |
| "learning_rate": 4.079579333738039e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 321 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.644, | |
| "grad_norm": 0.8479946255683899, | |
| "kl": 0.0014972686767578125, | |
| "learning_rate": 4.0498043714627006e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 322 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.646, | |
| "grad_norm": 0.7696375250816345, | |
| "kl": 0.002063751220703125, | |
| "learning_rate": 4.020100089676376e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 323 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.648, | |
| "grad_norm": 0.012891444377601147, | |
| "kl": 0.00150299072265625, | |
| "learning_rate": 3.9904679361238526e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 324 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.65, | |
| "grad_norm": 1.1341594457626343, | |
| "kl": 0.002437591552734375, | |
| "learning_rate": 3.9609093550344907e-07, | |
| "loss": 0.0001, | |
| "reward": 0.375, | |
| "reward_std": 0.5303300842642784, | |
| "rewards/accuracy_reward": 0.25, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 325 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.652, | |
| "grad_norm": 0.015124933794140816, | |
| "kl": 0.0007419586181640625, | |
| "learning_rate": 3.931425787051832e-07, | |
| "loss": 0.0, | |
| "reward": 0.75, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 326 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.654, | |
| "grad_norm": 0.7730603814125061, | |
| "kl": 0.002941131591796875, | |
| "learning_rate": 3.902018669163384e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 327 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.656, | |
| "grad_norm": 0.02122451364994049, | |
| "kl": 0.000957489013671875, | |
| "learning_rate": 3.872689434630585e-07, | |
| "loss": 0.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 328 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.658, | |
| "grad_norm": 0.01871700957417488, | |
| "kl": 0.0010528564453125, | |
| "learning_rate": 3.843439512918949e-07, | |
| "loss": 0.0, | |
| "reward": 0.75, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 329 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.66, | |
| "grad_norm": 0.0117810582742095, | |
| "kl": 0.0014801025390625, | |
| "learning_rate": 3.8142703296283953e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 330 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1353.5, | |
| "epoch": 0.662, | |
| "grad_norm": 0.017292601987719536, | |
| "kl": 0.0030651092529296875, | |
| "learning_rate": 3.785183306423767e-07, | |
| "loss": 0.0001, | |
| "reward": 0.5, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.5, | |
| "step": 331 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1955.25, | |
| "epoch": 0.664, | |
| "grad_norm": 0.7610009908676147, | |
| "kl": 0.001689910888671875, | |
| "learning_rate": 3.7561798609655373e-07, | |
| "loss": 0.0353, | |
| "reward": 0.25, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 332 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.666, | |
| "grad_norm": 0.013269172981381416, | |
| "kl": 0.0008087158203125, | |
| "learning_rate": 3.72726140684072e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 333 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.668, | |
| "grad_norm": 0.01594419591128826, | |
| "kl": 0.000812530517578125, | |
| "learning_rate": 3.6984293534939737e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 334 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.67, | |
| "grad_norm": 0.010347824543714523, | |
| "kl": 0.000751495361328125, | |
| "learning_rate": 3.6696851061588994e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 335 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.672, | |
| "grad_norm": 0.014122740365564823, | |
| "kl": 0.0012054443359375, | |
| "learning_rate": 3.641030065789562e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 336 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.674, | |
| "grad_norm": 0.013101037591695786, | |
| "kl": 0.001033782958984375, | |
| "learning_rate": 3.612465628992203e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 337 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.676, | |
| "grad_norm": 0.666260838508606, | |
| "kl": 0.00324249267578125, | |
| "learning_rate": 3.5839931879571725e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 338 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2022.75, | |
| "epoch": 0.678, | |
| "grad_norm": 0.7250146269798279, | |
| "kl": 0.00144195556640625, | |
| "learning_rate": 3.555614130391079e-07, | |
| "loss": 0.009, | |
| "reward": 0.375, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.375, | |
| "step": 339 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2003.25, | |
| "epoch": 0.68, | |
| "grad_norm": 0.7383328676223755, | |
| "kl": 0.00299072265625, | |
| "learning_rate": 3.5273298394491515e-07, | |
| "loss": -0.016, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 340 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.682, | |
| "grad_norm": 0.011795282363891602, | |
| "kl": 0.000850677490234375, | |
| "learning_rate": 3.4991416936678276e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 341 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.684, | |
| "grad_norm": 0.9384401440620422, | |
| "kl": 0.002079010009765625, | |
| "learning_rate": 3.471051066897562e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 342 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.686, | |
| "grad_norm": 0.010377887636423111, | |
| "kl": 0.0008144378662109375, | |
| "learning_rate": 3.4430593282358777e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 343 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.688, | |
| "grad_norm": 0.7026439309120178, | |
| "kl": 0.001007080078125, | |
| "learning_rate": 3.4151678419606233e-07, | |
| "loss": 0.0, | |
| "reward": 0.375, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.375, | |
| "step": 344 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.69, | |
| "grad_norm": 0.011138451285660267, | |
| "kl": 0.0008373260498046875, | |
| "learning_rate": 3.387377967463493e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 345 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.692, | |
| "grad_norm": 0.7246440649032593, | |
| "kl": 0.0035762786865234375, | |
| "learning_rate": 3.359691059183761e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 346 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.694, | |
| "grad_norm": 0.01353926956653595, | |
| "kl": 0.001087188720703125, | |
| "learning_rate": 3.3321084665422803e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 347 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1851.25, | |
| "epoch": 0.696, | |
| "grad_norm": 1.2388075590133667, | |
| "kl": 0.00121307373046875, | |
| "learning_rate": 3.3046315338757026e-07, | |
| "loss": 0.0841, | |
| "reward": 0.1875, | |
| "reward_std": 0.2651650384068489, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 348 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2005.75, | |
| "epoch": 0.698, | |
| "grad_norm": 0.7063978314399719, | |
| "kl": 0.002201080322265625, | |
| "learning_rate": 3.2772616003709616e-07, | |
| "loss": 0.0153, | |
| "reward": 0.375, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.375, | |
| "step": 349 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.7, | |
| "grad_norm": 0.009749515913426876, | |
| "kl": 0.001163482666015625, | |
| "learning_rate": 3.250000000000001e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 350 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.702, | |
| "grad_norm": 0.01226428709924221, | |
| "kl": 0.00107574462890625, | |
| "learning_rate": 3.222848061454764e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 351 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.704, | |
| "grad_norm": 0.01303025335073471, | |
| "kl": 0.001399993896484375, | |
| "learning_rate": 3.195807108082429e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 352 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.706, | |
| "grad_norm": 0.028528396040201187, | |
| "kl": 0.001117706298828125, | |
| "learning_rate": 3.168878457820915e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 353 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1785.25, | |
| "epoch": 0.708, | |
| "grad_norm": 0.9730527997016907, | |
| "kl": 0.001430511474609375, | |
| "learning_rate": 3.142063423134644e-07, | |
| "loss": 0.122, | |
| "reward": 0.875, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.375, | |
| "step": 354 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.71, | |
| "grad_norm": 0.013550005853176117, | |
| "kl": 0.001338958740234375, | |
| "learning_rate": 3.115363310950578e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 355 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.712, | |
| "grad_norm": 0.010767250321805477, | |
| "kl": 0.00067138671875, | |
| "learning_rate": 3.0887794225945143e-07, | |
| "loss": 0.0, | |
| "reward": 0.75, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 356 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.714, | |
| "grad_norm": 0.012552580796182156, | |
| "kl": 0.00140380859375, | |
| "learning_rate": 3.062313053727671e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 357 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.716, | |
| "grad_norm": 0.6516157984733582, | |
| "kl": 0.00159454345703125, | |
| "learning_rate": 3.0359654942835247e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 358 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.718, | |
| "grad_norm": 0.012717018835246563, | |
| "kl": 0.0008392333984375, | |
| "learning_rate": 3.0097380284049523e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 359 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.72, | |
| "grad_norm": 0.014254845678806305, | |
| "kl": 0.001018524169921875, | |
| "learning_rate": 2.9836319343816397e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 360 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.722, | |
| "grad_norm": 0.017017841339111328, | |
| "kl": 0.0009784698486328125, | |
| "learning_rate": 2.9576484845877793e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 361 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1976.75, | |
| "epoch": 0.724, | |
| "grad_norm": 0.8839628100395203, | |
| "kl": 0.002727508544921875, | |
| "learning_rate": 2.931788945420058e-07, | |
| "loss": 0.0265, | |
| "reward": 0.3125, | |
| "reward_std": 0.2651650309562683, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.3125, | |
| "step": 362 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.726, | |
| "grad_norm": 0.010562002658843994, | |
| "kl": 0.00103759765625, | |
| "learning_rate": 2.9060545772359305e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 363 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.728, | |
| "grad_norm": 0.013268781825900078, | |
| "kl": 0.0009174346923828125, | |
| "learning_rate": 2.8804466342921987e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 364 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1927.75, | |
| "epoch": 0.73, | |
| "grad_norm": 0.8109666705131531, | |
| "kl": 0.0020313262939453125, | |
| "learning_rate": 2.854966364683872e-07, | |
| "loss": 0.0471, | |
| "reward": 0.875, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.375, | |
| "step": 365 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.732, | |
| "grad_norm": 0.012277526780962944, | |
| "kl": 0.000827789306640625, | |
| "learning_rate": 2.829615010283344e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 366 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.734, | |
| "grad_norm": 0.7054362893104553, | |
| "kl": 0.0069103240966796875, | |
| "learning_rate": 2.8043938066798645e-07, | |
| "loss": 0.0003, | |
| "reward": 0.6875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 367 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.736, | |
| "grad_norm": 0.012917861342430115, | |
| "kl": 0.0012969970703125, | |
| "learning_rate": 2.7793039831193133e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 368 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.738, | |
| "grad_norm": 0.011064048856496811, | |
| "kl": 0.0007419586181640625, | |
| "learning_rate": 2.7543467624442956e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 369 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.74, | |
| "grad_norm": 0.014153026975691319, | |
| "kl": 0.00079345703125, | |
| "learning_rate": 2.729523361034538e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 370 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1957.5, | |
| "epoch": 0.742, | |
| "grad_norm": 0.02183438278734684, | |
| "kl": 0.002010345458984375, | |
| "learning_rate": 2.7048349887476037e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 371 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1967.0, | |
| "epoch": 0.744, | |
| "grad_norm": 0.6841965317726135, | |
| "kl": 0.00157928466796875, | |
| "learning_rate": 2.6802828488599294e-07, | |
| "loss": 0.0304, | |
| "reward": 0.375, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.375, | |
| "step": 372 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.746, | |
| "grad_norm": 0.010570400394499302, | |
| "kl": 0.00075531005859375, | |
| "learning_rate": 2.655868138008171e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 373 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.748, | |
| "grad_norm": 0.013351581990718842, | |
| "kl": 0.000873565673828125, | |
| "learning_rate": 2.631592046130896e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 374 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1565.75, | |
| "epoch": 0.75, | |
| "grad_norm": 0.9974377751350403, | |
| "kl": 0.008136749267578125, | |
| "learning_rate": 2.6074557564105724e-07, | |
| "loss": -0.3145, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 375 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1881.75, | |
| "epoch": 0.752, | |
| "grad_norm": 1.1940348148345947, | |
| "kl": 0.0038909912109375, | |
| "learning_rate": 2.583460445215911e-07, | |
| "loss": 0.0687, | |
| "reward": 0.3125, | |
| "reward_std": 0.2651650384068489, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.3125, | |
| "step": 376 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.754, | |
| "grad_norm": 0.01335156336426735, | |
| "kl": 0.0006580352783203125, | |
| "learning_rate": 2.5596072820445254e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 377 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.756, | |
| "grad_norm": 0.011012405157089233, | |
| "kl": 0.00142669677734375, | |
| "learning_rate": 2.5358974294659373e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 378 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.758, | |
| "grad_norm": 0.16783788800239563, | |
| "kl": 0.00443267822265625, | |
| "learning_rate": 2.512332043064913e-07, | |
| "loss": 0.0002, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 379 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.76, | |
| "grad_norm": 0.011893996968865395, | |
| "kl": 0.001003265380859375, | |
| "learning_rate": 2.488912271385139e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 380 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.762, | |
| "grad_norm": 0.7219942212104797, | |
| "kl": 0.00760650634765625, | |
| "learning_rate": 2.465639255873246e-07, | |
| "loss": 0.0003, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 381 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.764, | |
| "grad_norm": 0.04678433761000633, | |
| "kl": 0.0009021759033203125, | |
| "learning_rate": 2.4425141308231765e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 382 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.766, | |
| "grad_norm": 0.011990766040980816, | |
| "kl": 0.0006542205810546875, | |
| "learning_rate": 2.4195380233209006e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 383 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.768, | |
| "grad_norm": 0.011609113775193691, | |
| "kl": 0.0009002685546875, | |
| "learning_rate": 2.3967120531894857e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 384 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.77, | |
| "grad_norm": 0.029996510595083237, | |
| "kl": 0.001007080078125, | |
| "learning_rate": 2.374037332934512e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 385 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.772, | |
| "grad_norm": 0.7205003499984741, | |
| "kl": 0.001007080078125, | |
| "learning_rate": 2.3515149676898552e-07, | |
| "loss": 0.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 386 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.774, | |
| "grad_norm": 0.01278019044548273, | |
| "kl": 0.000644683837890625, | |
| "learning_rate": 2.3291460551638237e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 387 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.776, | |
| "grad_norm": 0.050757694989442825, | |
| "kl": 0.0013141632080078125, | |
| "learning_rate": 2.306931685585657e-07, | |
| "loss": 0.0001, | |
| "reward": 0.75, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 388 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.778, | |
| "grad_norm": 0.8608619570732117, | |
| "kl": 0.00222015380859375, | |
| "learning_rate": 2.2848729416523859e-07, | |
| "loss": 0.0001, | |
| "reward": 0.625, | |
| "reward_std": 0.5303300768136978, | |
| "rewards/accuracy_reward": 0.25, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.375, | |
| "step": 389 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.78, | |
| "grad_norm": 0.01890842616558075, | |
| "kl": 0.0016078948974609375, | |
| "learning_rate": 2.2629708984760706e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 390 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1757.5, | |
| "epoch": 0.782, | |
| "grad_norm": 0.9615358710289001, | |
| "kl": 0.00257110595703125, | |
| "learning_rate": 2.2412266235313973e-07, | |
| "loss": 0.1401, | |
| "reward": 0.375, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.375, | |
| "step": 391 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2038.25, | |
| "epoch": 0.784, | |
| "grad_norm": 0.7946398854255676, | |
| "kl": 0.0009212493896484375, | |
| "learning_rate": 2.2196411766036487e-07, | |
| "loss": 0.0034, | |
| "reward": 0.8125, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.3125, | |
| "step": 392 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.786, | |
| "grad_norm": 0.01277601532638073, | |
| "kl": 0.001102447509765625, | |
| "learning_rate": 2.1982156097370557e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 393 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1800.5, | |
| "epoch": 0.788, | |
| "grad_norm": 0.9031627774238586, | |
| "kl": 0.00704193115234375, | |
| "learning_rate": 2.1769509671835223e-07, | |
| "loss": 0.1129, | |
| "reward": 0.375, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.375, | |
| "step": 394 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.79, | |
| "grad_norm": 0.017206581309437752, | |
| "kl": 0.001468658447265625, | |
| "learning_rate": 2.1558482853517253e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 395 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1820.75, | |
| "epoch": 0.792, | |
| "grad_norm": 0.9008931517601013, | |
| "kl": 0.0047149658203125, | |
| "learning_rate": 2.134908592756607e-07, | |
| "loss": 0.0002, | |
| "reward": 0.0625, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.0625, | |
| "step": 396 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2010.25, | |
| "epoch": 0.794, | |
| "grad_norm": 0.8031813502311707, | |
| "kl": 0.00140380859375, | |
| "learning_rate": 2.1141329099692406e-07, | |
| "loss": -0.0135, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 397 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.796, | |
| "grad_norm": 0.015710551291704178, | |
| "kl": 0.0016117095947265625, | |
| "learning_rate": 2.0935222495670968e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 398 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.798, | |
| "grad_norm": 0.7256066203117371, | |
| "kl": 0.00640869140625, | |
| "learning_rate": 2.0730776160846853e-07, | |
| "loss": 0.0003, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 399 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.8, | |
| "grad_norm": 0.011895825155079365, | |
| "kl": 0.00131988525390625, | |
| "learning_rate": 2.0528000059645995e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 400 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.802, | |
| "grad_norm": 0.01262570358812809, | |
| "kl": 0.00177001953125, | |
| "learning_rate": 2.032690407508949e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 401 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.804, | |
| "grad_norm": 0.014729364775121212, | |
| "kl": 0.00090789794921875, | |
| "learning_rate": 2.0127498008311922e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 402 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.806, | |
| "grad_norm": 0.012951449491083622, | |
| "kl": 0.00146484375, | |
| "learning_rate": 1.9929791578083655e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 403 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.808, | |
| "grad_norm": 0.6974371671676636, | |
| "kl": 0.0015716552734375, | |
| "learning_rate": 1.9733794420337213e-07, | |
| "loss": 0.0001, | |
| "reward": 0.375, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.375, | |
| "step": 404 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.81, | |
| "grad_norm": 0.012584330514073372, | |
| "kl": 0.001598358154296875, | |
| "learning_rate": 1.9539516087697517e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 405 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.812, | |
| "grad_norm": 0.011754573322832584, | |
| "kl": 0.00107574462890625, | |
| "learning_rate": 1.934696604901642e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 406 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.814, | |
| "grad_norm": 0.02457410842180252, | |
| "kl": 0.0015869140625, | |
| "learning_rate": 1.915615368891117e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 407 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.816, | |
| "grad_norm": 0.7841270565986633, | |
| "kl": 0.00121307373046875, | |
| "learning_rate": 1.8967088307307e-07, | |
| "loss": 0.0, | |
| "reward": 0.375, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.375, | |
| "step": 408 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.818, | |
| "grad_norm": 0.01260992232710123, | |
| "kl": 0.0014495849609375, | |
| "learning_rate": 1.8779779118983867e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 409 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.82, | |
| "grad_norm": 0.011464129202067852, | |
| "kl": 0.0010471343994140625, | |
| "learning_rate": 1.8594235253127372e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 410 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1915.0, | |
| "epoch": 0.822, | |
| "grad_norm": 0.8047151565551758, | |
| "kl": 0.00250244140625, | |
| "learning_rate": 1.8410465752883758e-07, | |
| "loss": -0.0527, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 411 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.824, | |
| "grad_norm": 0.0124558350071311, | |
| "kl": 0.00130462646484375, | |
| "learning_rate": 1.822847957491922e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 412 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.826, | |
| "grad_norm": 0.01380992028862238, | |
| "kl": 0.0008144378662109375, | |
| "learning_rate": 1.804828558898332e-07, | |
| "loss": 0.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 413 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.828, | |
| "grad_norm": 0.9551144242286682, | |
| "kl": 0.011430740356445312, | |
| "learning_rate": 1.7869892577476722e-07, | |
| "loss": 0.0005, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 414 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.83, | |
| "grad_norm": 0.012345471419394016, | |
| "kl": 0.001438140869140625, | |
| "learning_rate": 1.7693309235023127e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 415 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1821.0, | |
| "epoch": 0.832, | |
| "grad_norm": 0.7977136969566345, | |
| "kl": 0.001773834228515625, | |
| "learning_rate": 1.7518544168045524e-07, | |
| "loss": 0.0981, | |
| "reward": 0.375, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.375, | |
| "step": 416 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.834, | |
| "grad_norm": 0.011019429191946983, | |
| "kl": 0.0006771087646484375, | |
| "learning_rate": 1.7345605894346726e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 417 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.836, | |
| "grad_norm": 0.015206689946353436, | |
| "kl": 0.0016021728515625, | |
| "learning_rate": 1.7174502842694212e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 418 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.838, | |
| "grad_norm": 0.04012497141957283, | |
| "kl": 0.000911712646484375, | |
| "learning_rate": 1.7005243352409333e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 419 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.84, | |
| "grad_norm": 0.01337823923677206, | |
| "kl": 0.0009021759033203125, | |
| "learning_rate": 1.6837835672960831e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 420 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.842, | |
| "grad_norm": 0.010002349503338337, | |
| "kl": 0.0014495849609375, | |
| "learning_rate": 1.6672287963562852e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 421 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.844, | |
| "grad_norm": 0.716344952583313, | |
| "kl": 0.00653076171875, | |
| "learning_rate": 1.6508608292777203e-07, | |
| "loss": 0.0003, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 422 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.846, | |
| "grad_norm": 0.037377193570137024, | |
| "kl": 0.001346588134765625, | |
| "learning_rate": 1.6346804638120098e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 423 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.848, | |
| "grad_norm": 0.011378524824976921, | |
| "kl": 0.0007991790771484375, | |
| "learning_rate": 1.6186884885673413e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 424 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.85, | |
| "grad_norm": 0.012089181691408157, | |
| "kl": 0.00154876708984375, | |
| "learning_rate": 1.6028856829700258e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 425 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.852, | |
| "grad_norm": 0.7250344753265381, | |
| "kl": 0.002223968505859375, | |
| "learning_rate": 1.5872728172265146e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 426 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1880.5, | |
| "epoch": 0.854, | |
| "grad_norm": 0.8333255648612976, | |
| "kl": 0.0015106201171875, | |
| "learning_rate": 1.5718506522858572e-07, | |
| "loss": 0.0692, | |
| "reward": 0.375, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.375, | |
| "step": 427 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1326.75, | |
| "epoch": 0.856, | |
| "grad_norm": 1.4510780572891235, | |
| "kl": 0.0024261474609375, | |
| "learning_rate": 1.5566199398026147e-07, | |
| "loss": 0.0319, | |
| "reward": 0.375, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.375, | |
| "step": 428 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.858, | |
| "grad_norm": 0.01227201521396637, | |
| "kl": 0.0006866455078125, | |
| "learning_rate": 1.5415814221002265e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 429 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.86, | |
| "grad_norm": 0.012068121694028378, | |
| "kl": 0.0009098052978515625, | |
| "learning_rate": 1.5267358321348285e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 430 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.862, | |
| "grad_norm": 0.012486455962061882, | |
| "kl": 0.000980377197265625, | |
| "learning_rate": 1.5120838934595337e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 431 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.864, | |
| "grad_norm": 0.013689450919628143, | |
| "kl": 0.00124359130859375, | |
| "learning_rate": 1.4976263201891613e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 432 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.866, | |
| "grad_norm": 0.5863283276557922, | |
| "kl": 0.00223541259765625, | |
| "learning_rate": 1.483363816965435e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 433 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.868, | |
| "grad_norm": 0.01356459315866232, | |
| "kl": 0.00078582763671875, | |
| "learning_rate": 1.469297078922642e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 434 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.87, | |
| "grad_norm": 0.009759028442203999, | |
| "kl": 0.0011749267578125, | |
| "learning_rate": 1.4554267916537495e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 435 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.872, | |
| "grad_norm": 0.02292313612997532, | |
| "kl": 0.0009002685546875, | |
| "learning_rate": 1.4417536311769885e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 436 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.874, | |
| "grad_norm": 0.013289421796798706, | |
| "kl": 0.00077056884765625, | |
| "learning_rate": 1.4282782639029128e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 437 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.876, | |
| "grad_norm": 0.0265584085136652, | |
| "kl": 0.00119781494140625, | |
| "learning_rate": 1.4150013466019114e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 438 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2008.25, | |
| "epoch": 0.878, | |
| "grad_norm": 0.013609832152724266, | |
| "kl": 0.000782012939453125, | |
| "learning_rate": 1.4019235263722034e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 439 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.88, | |
| "grad_norm": 0.010987906716763973, | |
| "kl": 0.00115203857421875, | |
| "learning_rate": 1.3890454406082956e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 440 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.882, | |
| "grad_norm": 0.01149643212556839, | |
| "kl": 0.0007152557373046875, | |
| "learning_rate": 1.3763677169699217e-07, | |
| "loss": 0.0, | |
| "reward": 0.75, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 441 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.884, | |
| "grad_norm": 0.012063896283507347, | |
| "kl": 0.000858306884765625, | |
| "learning_rate": 1.3638909733514452e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 442 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.886, | |
| "grad_norm": 0.6539027094841003, | |
| "kl": 0.0033817291259765625, | |
| "learning_rate": 1.351615817851748e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 443 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.888, | |
| "grad_norm": 0.683651864528656, | |
| "kl": 0.001430511474609375, | |
| "learning_rate": 1.3395428487445914e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 444 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.89, | |
| "grad_norm": 0.014244834892451763, | |
| "kl": 0.001583099365234375, | |
| "learning_rate": 1.3276726544494571e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 445 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1551.25, | |
| "epoch": 0.892, | |
| "grad_norm": 0.04096902534365654, | |
| "kl": 0.003017425537109375, | |
| "learning_rate": 1.316005813502869e-07, | |
| "loss": 0.0001, | |
| "reward": 0.5, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.5, | |
| "step": 446 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.894, | |
| "grad_norm": 0.013455760665237904, | |
| "kl": 0.00115203857421875, | |
| "learning_rate": 1.3045428945301953e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 447 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1822.0, | |
| "epoch": 0.896, | |
| "grad_norm": 1.1271547079086304, | |
| "kl": 0.0069122314453125, | |
| "learning_rate": 1.2932844562179352e-07, | |
| "loss": -0.0998, | |
| "reward": 0.125, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 448 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.898, | |
| "grad_norm": 0.6208535432815552, | |
| "kl": 0.0009765625, | |
| "learning_rate": 1.2822310472864885e-07, | |
| "loss": 0.0, | |
| "reward": 0.5, | |
| "reward_std": 0.3535533845424652, | |
| "rewards/accuracy_reward": 0.25, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 449 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.9, | |
| "grad_norm": 0.009892701171338558, | |
| "kl": 0.0006608963012695312, | |
| "learning_rate": 1.2713832064634125e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 450 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.902, | |
| "grad_norm": 0.0119470888748765, | |
| "kl": 0.000751495361328125, | |
| "learning_rate": 1.260741462457165e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 451 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1717.0, | |
| "epoch": 0.904, | |
| "grad_norm": 0.015732314437627792, | |
| "kl": 0.001445770263671875, | |
| "learning_rate": 1.2503063339313356e-07, | |
| "loss": 0.0, | |
| "reward": 0.125, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 452 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.906, | |
| "grad_norm": 0.7161591649055481, | |
| "kl": 0.001621246337890625, | |
| "learning_rate": 1.2400783294793668e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 453 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.908, | |
| "grad_norm": 0.009790318086743355, | |
| "kl": 0.0006418228149414062, | |
| "learning_rate": 1.2300579475997657e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 454 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.91, | |
| "grad_norm": 0.01178540289402008, | |
| "kl": 0.0012798309326171875, | |
| "learning_rate": 1.220245676671809e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 455 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1967.75, | |
| "epoch": 0.912, | |
| "grad_norm": 0.918179452419281, | |
| "kl": 0.00170135498046875, | |
| "learning_rate": 1.2106419949317388e-07, | |
| "loss": 0.0301, | |
| "reward": 0.375, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.375, | |
| "step": 456 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.914, | |
| "grad_norm": 0.7712357640266418, | |
| "kl": 0.00215911865234375, | |
| "learning_rate": 1.2012473704494537e-07, | |
| "loss": 0.0001, | |
| "reward": 0.6875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 457 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1705.5, | |
| "epoch": 0.916, | |
| "grad_norm": 0.8978808522224426, | |
| "kl": 0.00180816650390625, | |
| "learning_rate": 1.1920622611056974e-07, | |
| "loss": 0.0779, | |
| "reward": 0.25, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 458 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.918, | |
| "grad_norm": 0.8227681517601013, | |
| "kl": 0.00136566162109375, | |
| "learning_rate": 1.1830871145697412e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 459 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.92, | |
| "grad_norm": 0.009116187691688538, | |
| "kl": 0.00128936767578125, | |
| "learning_rate": 1.1743223682775649e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 460 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.922, | |
| "grad_norm": 0.014834447763860226, | |
| "kl": 0.00095367431640625, | |
| "learning_rate": 1.1657684494105386e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 461 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.924, | |
| "grad_norm": 0.013181531801819801, | |
| "kl": 0.0008907318115234375, | |
| "learning_rate": 1.1574257748745986e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 462 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.926, | |
| "grad_norm": 0.010957694612443447, | |
| "kl": 0.0006504058837890625, | |
| "learning_rate": 1.1492947512799328e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 463 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.928, | |
| "grad_norm": 0.8815539479255676, | |
| "kl": 0.0010204315185546875, | |
| "learning_rate": 1.1413757749211602e-07, | |
| "loss": 0.0, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 464 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.93, | |
| "grad_norm": 0.010624675080180168, | |
| "kl": 0.0008087158203125, | |
| "learning_rate": 1.1336692317580158e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 465 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.932, | |
| "grad_norm": 0.013844382017850876, | |
| "kl": 0.001033782958984375, | |
| "learning_rate": 1.1261754973965422e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 466 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.934, | |
| "grad_norm": 0.012338408268988132, | |
| "kl": 0.000690460205078125, | |
| "learning_rate": 1.1188949370707787e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 467 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1994.0, | |
| "epoch": 0.936, | |
| "grad_norm": 0.7254545092582703, | |
| "kl": 0.0010471343994140625, | |
| "learning_rate": 1.1118279056249653e-07, | |
| "loss": 0.0197, | |
| "reward": 0.375, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.375, | |
| "step": 468 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.938, | |
| "grad_norm": 0.01179241482168436, | |
| "kl": 0.0006885528564453125, | |
| "learning_rate": 1.1049747474962444e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 469 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.94, | |
| "grad_norm": 0.6890900135040283, | |
| "kl": 0.001373291015625, | |
| "learning_rate": 1.0983357966978745e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 470 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.942, | |
| "grad_norm": 0.5639128684997559, | |
| "kl": 0.00131988525390625, | |
| "learning_rate": 1.0919113768029517e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 471 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.944, | |
| "grad_norm": 0.012823720462620258, | |
| "kl": 0.001026153564453125, | |
| "learning_rate": 1.0857018009286381e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 472 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.946, | |
| "grad_norm": 0.009964537806808949, | |
| "kl": 0.0007228851318359375, | |
| "learning_rate": 1.0797073717209013e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 473 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.948, | |
| "grad_norm": 0.7185282707214355, | |
| "kl": 0.0028629302978515625, | |
| "learning_rate": 1.0739283813397639e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 474 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.95, | |
| "grad_norm": 0.012274986132979393, | |
| "kl": 0.00087738037109375, | |
| "learning_rate": 1.068365111445064e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 475 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1744.25, | |
| "epoch": 0.952, | |
| "grad_norm": 0.8758648633956909, | |
| "kl": 0.002716064453125, | |
| "learning_rate": 1.063017833182728e-07, | |
| "loss": 0.1492, | |
| "reward": 0.1875, | |
| "reward_std": 0.2651650309562683, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 476 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.954, | |
| "grad_norm": 0.014645918272435665, | |
| "kl": 0.000720977783203125, | |
| "learning_rate": 1.0578868071715544e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 477 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.956, | |
| "grad_norm": 0.013319121673703194, | |
| "kl": 0.0009250640869140625, | |
| "learning_rate": 1.0529722834905125e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 478 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.958, | |
| "grad_norm": 0.009549077600240707, | |
| "kl": 0.0014190673828125, | |
| "learning_rate": 1.0482745016665526e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 479 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.96, | |
| "grad_norm": 0.015292295254766941, | |
| "kl": 0.0009307861328125, | |
| "learning_rate": 1.0437936906629334e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 480 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.962, | |
| "grad_norm": 1.031322717666626, | |
| "kl": 0.003509521484375, | |
| "learning_rate": 1.0395300688680625e-07, | |
| "loss": 0.0001, | |
| "reward": 0.125, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.125, | |
| "step": 481 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.964, | |
| "grad_norm": 0.011882193386554718, | |
| "kl": 0.001026153564453125, | |
| "learning_rate": 1.0354838440848501e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 482 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1847.0, | |
| "epoch": 0.966, | |
| "grad_norm": 0.6594410538673401, | |
| "kl": 0.002811431884765625, | |
| "learning_rate": 1.0316552135205837e-07, | |
| "loss": 0.0864, | |
| "reward": 0.4375, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.4375, | |
| "step": 483 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.968, | |
| "grad_norm": 0.013277575373649597, | |
| "kl": 0.000789642333984375, | |
| "learning_rate": 1.0280443637773163e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 484 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.97, | |
| "grad_norm": 0.7425467371940613, | |
| "kl": 0.00323486328125, | |
| "learning_rate": 1.0246514708427701e-07, | |
| "loss": 0.0001, | |
| "reward": 0.0625, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.0625, | |
| "step": 485 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.972, | |
| "grad_norm": 0.01078079268336296, | |
| "kl": 0.00154876708984375, | |
| "learning_rate": 1.0214767000817596e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 486 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.974, | |
| "grad_norm": 0.012354187667369843, | |
| "kl": 0.0005855560302734375, | |
| "learning_rate": 1.0185202062281336e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 487 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.976, | |
| "grad_norm": 0.7250139117240906, | |
| "kl": 0.0037212371826171875, | |
| "learning_rate": 1.0157821333772304e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 488 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.978, | |
| "grad_norm": 0.014553495682775974, | |
| "kl": 0.00141143798828125, | |
| "learning_rate": 1.013262614978859e-07, | |
| "loss": 0.0001, | |
| "reward": 0.75, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.5, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 489 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.98, | |
| "grad_norm": 0.01307358592748642, | |
| "kl": 0.000835418701171875, | |
| "learning_rate": 1.0109617738307911e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 490 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 1908.5, | |
| "epoch": 0.982, | |
| "grad_norm": 0.7941016554832458, | |
| "kl": 0.0010204315185546875, | |
| "learning_rate": 1.0088797220727779e-07, | |
| "loss": 0.0558, | |
| "reward": 0.375, | |
| "reward_std": 0.1767766922712326, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.375, | |
| "step": 491 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.984, | |
| "grad_norm": 0.008943392895162106, | |
| "kl": 0.000537872314453125, | |
| "learning_rate": 1.0070165611810855e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 492 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.986, | |
| "grad_norm": 0.012137016281485558, | |
| "kl": 0.001377105712890625, | |
| "learning_rate": 1.005372381963547e-07, | |
| "loss": 0.0001, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 493 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.988, | |
| "grad_norm": 0.01392819918692112, | |
| "kl": 0.0009784698486328125, | |
| "learning_rate": 1.0039472645551372e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 494 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.99, | |
| "grad_norm": 0.010369054041802883, | |
| "kl": 0.000942230224609375, | |
| "learning_rate": 1.002741278414069e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 495 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.992, | |
| "grad_norm": 0.011974328197538853, | |
| "kl": 0.0006275177001953125, | |
| "learning_rate": 1.0017544823184055e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 496 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.994, | |
| "grad_norm": 0.01267942413687706, | |
| "kl": 0.00104522705078125, | |
| "learning_rate": 1.0009869243631952e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 497 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.996, | |
| "grad_norm": 0.876571536064148, | |
| "kl": 0.003925323486328125, | |
| "learning_rate": 1.000438641958131e-07, | |
| "loss": 0.0002, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 498 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 0.998, | |
| "grad_norm": 0.6709615588188171, | |
| "kl": 0.001743316650390625, | |
| "learning_rate": 1.0001096618257236e-07, | |
| "loss": 0.0001, | |
| "reward": 0.1875, | |
| "reward_std": 0.0883883461356163, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.1875, | |
| "step": 499 | |
| }, | |
| { | |
| "clip_ratio": 0.0, | |
| "completion_length": 2048.0, | |
| "epoch": 1.0, | |
| "grad_norm": 0.014002146199345589, | |
| "kl": 0.0007781982421875, | |
| "learning_rate": 1e-07, | |
| "loss": 0.0, | |
| "reward": 0.25, | |
| "reward_std": 0.0, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/format_reward": 0.0, | |
| "rewards/tag_count_reward": 0.25, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "step": 500, | |
| "total_flos": 0.0, | |
| "train_loss": 0.0033100851627775683, | |
| "train_runtime": 14646.384, | |
| "train_samples_per_second": 0.068, | |
| "train_steps_per_second": 0.034 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 500, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 500, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 2, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |