diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,6760 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.12, + "eval_steps": 50, + "global_step": 300, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0004, + "frac_reward_zero_std": 0.75, + "grad_norm": 31.350610733032227, + "kl": 0.0, + "learning_rate": 0.0, + "loss": -0.0, + "num_tokens": 8880.0, + "reward": 0.8125, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.8125, + "rewards/compute_reward/std": 0.40311288833618164, + "step": 1 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0008, + "frac_reward_zero_std": 0.75, + "grad_norm": 40.88920593261719, + "kl": 0.0, + "learning_rate": 4e-08, + "loss": 0.0, + "num_tokens": 17760.0, + "reward": 0.375, + "reward_std": 0.14433756470680237, + "rewards/compute_reward/mean": 0.375, + "rewards/compute_reward/std": 0.5, + "step": 2 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0012, + "frac_reward_zero_std": 0.75, + "grad_norm": 41.24170684814453, + "kl": 0.0013443923089653254, + "learning_rate": 8e-08, + "loss": 0.0, + "num_tokens": 26640.0, + "reward": 0.3125, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.3125, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 3 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0016, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.025073383003473282, + "kl": 0.0012363345449557528, + "learning_rate": 1.2e-07, + "loss": 0.0, + "num_tokens": 35520.0, + "reward": 0.25, + "reward_std": 0.0, + "rewards/compute_reward/mean": 0.25, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 4 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.002, + "frac_reward_zero_std": 0.5, + "grad_norm": 45.65277862548828, + "kl": 0.0009983214986277744, + "learning_rate": 1.6e-07, + "loss": 0.0, + "num_tokens": 44400.0, + "reward": 0.625, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.625, + "rewards/compute_reward/std": 0.5, + "step": 5 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0024, + "frac_reward_zero_std": 0.5, + "grad_norm": 40.60847854614258, + "kl": 0.0010500029457034543, + "learning_rate": 2e-07, + "loss": 0.0, + "num_tokens": 53280.0, + "reward": 0.5, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 6 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0028, + "frac_reward_zero_std": 0.5, + "grad_norm": 37.455440521240234, + "kl": 0.0013627598236780614, + "learning_rate": 2.4e-07, + "loss": 0.0, + "num_tokens": 62160.0, + "reward": 0.5625, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.5625, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 7 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0032, + "frac_reward_zero_std": 0.5, + "grad_norm": 49.363807678222656, + "kl": 0.0019289999618194997, + "learning_rate": 2.8e-07, + "loss": 0.0, + "num_tokens": 71040.0, + "reward": 0.4375, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.4375, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 8 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0036, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.055168308317661285, + "kl": 0.0023864044342190027, + "learning_rate": 3.2e-07, + "loss": 0.0, + "num_tokens": 79920.0, + "reward": 0.5, + "reward_std": 0.0, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 9 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.004, + "frac_reward_zero_std": 0.75, + "grad_norm": 27.4397029876709, + "kl": 0.0029534806380979717, + "learning_rate": 3.6e-07, + "loss": 0.0, + "num_tokens": 88800.0, + "reward": 0.3125, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.3125, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 10 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0044, + "frac_reward_zero_std": 0.5, + "grad_norm": 34.00825881958008, + "kl": 0.004601971711963415, + "learning_rate": 4e-07, + "loss": 0.0, + "num_tokens": 97680.0, + "reward": 0.1875, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.1875, + "rewards/compute_reward/std": 0.40311288833618164, + "step": 11 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0048, + "frac_reward_zero_std": 0.5, + "grad_norm": 41.115177154541016, + "kl": 0.005590100074186921, + "learning_rate": 4.3999999999999997e-07, + "loss": 0.0, + "num_tokens": 106560.0, + "reward": 0.5, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 12 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0052, + "frac_reward_zero_std": 0.5, + "grad_norm": 37.690364837646484, + "kl": 0.008083905442617834, + "learning_rate": 4.8e-07, + "loss": 0.0, + "num_tokens": 115440.0, + "reward": 0.25, + "reward_std": 0.28867512941360474, + "rewards/compute_reward/mean": 0.25, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 13 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0056, + "frac_reward_zero_std": 0.5, + "grad_norm": 44.36626052856445, + "kl": 0.01064743846654892, + "learning_rate": 5.2e-07, + "loss": 0.0, + "num_tokens": 124320.0, + "reward": 0.25, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.25, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 14 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.006, + "frac_reward_zero_std": 0.25, + "grad_norm": 35.651023864746094, + "kl": 0.01316027413122356, + "learning_rate": 5.6e-07, + "loss": 0.0001, + "num_tokens": 133200.0, + "reward": 0.75, + "reward_std": 0.39433756470680237, + "rewards/compute_reward/mean": 0.75, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 15 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0064, + "frac_reward_zero_std": 0.25, + "grad_norm": 52.94963836669922, + "kl": 0.020378376357257366, + "learning_rate": 6e-07, + "loss": 0.0001, + "num_tokens": 142080.0, + "reward": 0.4375, + "reward_std": 0.41367512941360474, + "rewards/compute_reward/mean": 0.4375, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 16 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0068, + "frac_reward_zero_std": 0.75, + "grad_norm": 36.93665313720703, + "kl": 0.025172261521220207, + "learning_rate": 6.4e-07, + "loss": 0.0001, + "num_tokens": 150960.0, + "reward": 0.3125, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.3125, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 17 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0072, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.5544130206108093, + "kl": 0.032024398911744356, + "learning_rate": 6.800000000000001e-07, + "loss": 0.0001, + "num_tokens": 159840.0, + "reward": 0.25, + "reward_std": 0.0, + "rewards/compute_reward/mean": 0.25, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 18 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0076, + "frac_reward_zero_std": 0.25, + "grad_norm": 44.66011047363281, + "kl": 0.039961141999810934, + "learning_rate": 7.2e-07, + "loss": 0.0002, + "num_tokens": 168720.0, + "reward": 0.25, + "reward_std": 0.39433756470680237, + "rewards/compute_reward/mean": 0.25, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 19 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.008, + "frac_reward_zero_std": 0.75, + "grad_norm": 28.504152297973633, + "kl": 0.036296515725553036, + "learning_rate": 7.599999999999999e-07, + "loss": 0.0001, + "num_tokens": 177600.0, + "reward": 0.9375, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.9375, + "rewards/compute_reward/std": 0.25, + "step": 20 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0084, + "frac_reward_zero_std": 0.25, + "grad_norm": 50.410888671875, + "kl": 0.04341549798846245, + "learning_rate": 8e-07, + "loss": 0.0002, + "num_tokens": 186480.0, + "reward": 0.375, + "reward_std": 0.39433756470680237, + "rewards/compute_reward/mean": 0.375, + "rewards/compute_reward/std": 0.5, + "step": 21 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0088, + "frac_reward_zero_std": 0.5, + "grad_norm": 51.3606071472168, + "kl": 0.050860715098679066, + "learning_rate": 8.399999999999999e-07, + "loss": 0.0002, + "num_tokens": 195360.0, + "reward": 0.5625, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.5625, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 22 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0092, + "frac_reward_zero_std": 0.75, + "grad_norm": 24.91278076171875, + "kl": 0.06213559303432703, + "learning_rate": 8.799999999999999e-07, + "loss": 0.0002, + "num_tokens": 204240.0, + "reward": 0.3125, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.3125, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 23 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0096, + "frac_reward_zero_std": 0.5, + "grad_norm": 39.62419891357422, + "kl": 0.0647846395149827, + "learning_rate": 9.2e-07, + "loss": 0.0003, + "num_tokens": 213120.0, + "reward": 0.625, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.625, + "rewards/compute_reward/std": 0.5, + "step": 24 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.01, + "frac_reward_zero_std": 0.25, + "grad_norm": 63.561767578125, + "kl": 0.0892756637185812, + "learning_rate": 9.6e-07, + "loss": 0.0004, + "num_tokens": 222000.0, + "reward": 0.5625, + "reward_std": 0.41367512941360474, + "rewards/compute_reward/mean": 0.5625, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 25 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0104, + "frac_reward_zero_std": 0.5, + "grad_norm": 42.179954528808594, + "kl": 0.10400468856096268, + "learning_rate": 1e-06, + "loss": 0.0004, + "num_tokens": 230880.0, + "reward": 0.5, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 26 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0108, + "frac_reward_zero_std": 0.75, + "grad_norm": 25.707307815551758, + "kl": 0.1136582437902689, + "learning_rate": 1.04e-06, + "loss": 0.0005, + "num_tokens": 239760.0, + "reward": 0.125, + "reward_std": 0.14433756470680237, + "rewards/compute_reward/mean": 0.125, + "rewards/compute_reward/std": 0.3415650427341461, + "step": 27 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0112, + "frac_reward_zero_std": 0.75, + "grad_norm": 26.41197967529297, + "kl": 0.13468273170292377, + "learning_rate": 1.08e-06, + "loss": 0.0005, + "num_tokens": 248640.0, + "reward": 0.5625, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.5625, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 28 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0116, + "frac_reward_zero_std": 0.75, + "grad_norm": 35.851348876953125, + "kl": 0.1477714516222477, + "learning_rate": 1.12e-06, + "loss": 0.0006, + "num_tokens": 257520.0, + "reward": 0.625, + "reward_std": 0.14433756470680237, + "rewards/compute_reward/mean": 0.625, + "rewards/compute_reward/std": 0.5, + "step": 29 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.012, + "frac_reward_zero_std": 0.75, + "grad_norm": 29.354488372802734, + "kl": 0.14891785010695457, + "learning_rate": 1.16e-06, + "loss": 0.0006, + "num_tokens": 266400.0, + "reward": 0.6875, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.6875, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 30 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0124, + "frac_reward_zero_std": 0.5, + "grad_norm": 39.174476623535156, + "kl": 0.15899883769452572, + "learning_rate": 1.2e-06, + "loss": 0.0006, + "num_tokens": 275280.0, + "reward": 0.25, + "reward_std": 0.28867512941360474, + "rewards/compute_reward/mean": 0.25, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 31 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0128, + "frac_reward_zero_std": 0.75, + "grad_norm": 44.24154281616211, + "kl": 0.22540221363306046, + "learning_rate": 1.24e-06, + "loss": 0.0009, + "num_tokens": 284160.0, + "reward": 0.5625, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.5625, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 32 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0132, + "frac_reward_zero_std": 0.5, + "grad_norm": 32.92380142211914, + "kl": 0.19463030248880386, + "learning_rate": 1.28e-06, + "loss": 0.0008, + "num_tokens": 293040.0, + "reward": 0.5, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 33 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0136, + "frac_reward_zero_std": 0.75, + "grad_norm": 32.65846633911133, + "kl": 0.2380228154361248, + "learning_rate": 1.32e-06, + "loss": 0.001, + "num_tokens": 301920.0, + "reward": 0.5625, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.5625, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 34 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.014, + "frac_reward_zero_std": 0.5, + "grad_norm": 77.5787582397461, + "kl": 0.21238164231181145, + "learning_rate": 1.3600000000000001e-06, + "loss": 0.0008, + "num_tokens": 310800.0, + "reward": 0.125, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.125, + "rewards/compute_reward/std": 0.3415650427341461, + "step": 35 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0144, + "frac_reward_zero_std": 0.75, + "grad_norm": 29.662382125854492, + "kl": 0.29322075098752975, + "learning_rate": 1.4e-06, + "loss": 0.0012, + "num_tokens": 319680.0, + "reward": 0.0625, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.0625, + "rewards/compute_reward/std": 0.25, + "step": 36 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0148, + "frac_reward_zero_std": 0.5, + "grad_norm": 48.11992263793945, + "kl": 0.25715991109609604, + "learning_rate": 1.44e-06, + "loss": 0.001, + "num_tokens": 328560.0, + "reward": 0.125, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.125, + "rewards/compute_reward/std": 0.3415650427341461, + "step": 37 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0152, + "frac_reward_zero_std": 0.75, + "grad_norm": 38.42544937133789, + "kl": 0.2538900189101696, + "learning_rate": 1.48e-06, + "loss": 0.001, + "num_tokens": 337440.0, + "reward": 0.125, + "reward_std": 0.14433756470680237, + "rewards/compute_reward/mean": 0.125, + "rewards/compute_reward/std": 0.3415650427341461, + "step": 38 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0156, + "frac_reward_zero_std": 0.0, + "grad_norm": 76.46257019042969, + "kl": 0.29158805310726166, + "learning_rate": 1.5199999999999998e-06, + "loss": 0.0012, + "num_tokens": 346320.0, + "reward": 0.4375, + "reward_std": 0.5580127239227295, + "rewards/compute_reward/mean": 0.4375, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 39 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.016, + "frac_reward_zero_std": 0.5, + "grad_norm": 50.54710388183594, + "kl": 0.42541511356830597, + "learning_rate": 1.5599999999999999e-06, + "loss": 0.0017, + "num_tokens": 355200.0, + "reward": 0.3125, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.3125, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 40 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0164, + "frac_reward_zero_std": 0.5, + "grad_norm": 47.023494720458984, + "kl": 0.3670000210404396, + "learning_rate": 1.6e-06, + "loss": 0.0015, + "num_tokens": 364080.0, + "reward": 0.3125, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.3125, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 41 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0168, + "frac_reward_zero_std": 0.75, + "grad_norm": 32.30452346801758, + "kl": 0.38483476638793945, + "learning_rate": 1.6399999999999998e-06, + "loss": 0.0015, + "num_tokens": 372960.0, + "reward": 0.0625, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.0625, + "rewards/compute_reward/std": 0.25, + "step": 42 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0172, + "frac_reward_zero_std": 0.5, + "grad_norm": 42.685611724853516, + "kl": 0.3910580351948738, + "learning_rate": 1.6799999999999998e-06, + "loss": 0.0016, + "num_tokens": 381840.0, + "reward": 0.1875, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.1875, + "rewards/compute_reward/std": 0.40311288833618164, + "step": 43 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0176, + "frac_reward_zero_std": 0.75, + "grad_norm": 39.6999626159668, + "kl": 0.4346514791250229, + "learning_rate": 1.7199999999999998e-06, + "loss": 0.0017, + "num_tokens": 390720.0, + "reward": 0.4375, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.4375, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 44 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.018, + "frac_reward_zero_std": 0.75, + "grad_norm": 32.37580490112305, + "kl": 0.38541124016046524, + "learning_rate": 1.7599999999999999e-06, + "loss": 0.0015, + "num_tokens": 399600.0, + "reward": 0.0625, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.0625, + "rewards/compute_reward/std": 0.25, + "step": 45 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0184, + "frac_reward_zero_std": 0.5, + "grad_norm": 37.88615798950195, + "kl": 0.3760378360748291, + "learning_rate": 1.8e-06, + "loss": 0.0015, + "num_tokens": 408480.0, + "reward": 0.375, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.375, + "rewards/compute_reward/std": 0.5, + "step": 46 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0188, + "frac_reward_zero_std": 0.5, + "grad_norm": 50.25699996948242, + "kl": 0.4850776717066765, + "learning_rate": 1.84e-06, + "loss": 0.0019, + "num_tokens": 417360.0, + "reward": 0.1875, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.1875, + "rewards/compute_reward/std": 0.40311288833618164, + "step": 47 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0192, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.5161222815513611, + "kl": 0.3658723160624504, + "learning_rate": 1.8799999999999998e-06, + "loss": 0.0015, + "num_tokens": 426240.0, + "reward": 0.25, + "reward_std": 0.0, + "rewards/compute_reward/mean": 0.25, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 48 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0196, + "frac_reward_zero_std": 0.5, + "grad_norm": 37.257171630859375, + "kl": 0.3248850479722023, + "learning_rate": 1.92e-06, + "loss": 0.0013, + "num_tokens": 435120.0, + "reward": 0.875, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.875, + "rewards/compute_reward/std": 0.3415650427341461, + "step": 49 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.02, + "frac_reward_zero_std": 0.5, + "grad_norm": 40.35070037841797, + "kl": 0.3114013075828552, + "learning_rate": 1.96e-06, + "loss": 0.0012, + "num_tokens": 444000.0, + "reward": 0.375, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.375, + "rewards/compute_reward/std": 0.5, + "step": 50 + }, + { + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.02, + "eval_loss": 0.0003695653285831213, + "eval_runtime": 12968.8828, + "eval_samples_per_second": 0.015, + "eval_steps_per_second": 0.004, + "frac_reward_zero_std": 0.54, + "num_tokens": 555000.0, + "reward": 0.445, + "reward_std": 0.23928203105926513, + "rewards/compute_reward/mean": 0.445, + "rewards/compute_reward/std": 0.23928203105926513, + "step": 50 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0204, + "frac_reward_zero_std": 0.75, + "grad_norm": 35.21977233886719, + "kl": 0.40479009598493576, + "learning_rate": 2e-06, + "loss": -0.1234, + "num_tokens": 563880.0, + "reward": 0.5625, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.5625, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 51 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0208, + "frac_reward_zero_std": 0.75, + "grad_norm": 17.721899032592773, + "kl": 0.3238118216395378, + "learning_rate": 1.9999756307053944e-06, + "loss": 0.1263, + "num_tokens": 572760.0, + "reward": 0.875, + "reward_std": 0.14433756470680237, + "rewards/compute_reward/mean": 0.875, + "rewards/compute_reward/std": 0.3415650427341461, + "step": 52 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0212, + "frac_reward_zero_std": 0.5, + "grad_norm": 48.17813491821289, + "kl": 0.4712698385119438, + "learning_rate": 1.999902524009304e-06, + "loss": -0.1231, + "num_tokens": 581640.0, + "reward": 0.25, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.25, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 53 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0216, + "frac_reward_zero_std": 1.0, + "grad_norm": 21.3551025390625, + "kl": 0.4696747958660126, + "learning_rate": 1.999780683474845e-06, + "loss": 0.1269, + "num_tokens": 590520.0, + "reward": 0.25, + "reward_std": 0.0, + "rewards/compute_reward/mean": 0.25, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 54 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.022, + "frac_reward_zero_std": 0.75, + "grad_norm": 42.38340377807617, + "kl": 0.5665004104375839, + "learning_rate": 1.999610115040354e-06, + "loss": 0.1272, + "num_tokens": 599400.0, + "reward": 0.6875, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.6875, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 55 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0224, + "frac_reward_zero_std": 0.5, + "grad_norm": 40.7006721496582, + "kl": 0.4810464084148407, + "learning_rate": 1.9993908270190957e-06, + "loss": -0.248, + "num_tokens": 608280.0, + "reward": 0.1875, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.1875, + "rewards/compute_reward/std": 0.40311288833618164, + "step": 56 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0228, + "frac_reward_zero_std": 0.25, + "grad_norm": 31.320539474487305, + "kl": 0.44268205761909485, + "learning_rate": 1.999122830098858e-06, + "loss": 0.7516, + "num_tokens": 617160.0, + "reward": 0.3125, + "reward_std": 0.375, + "rewards/compute_reward/mean": 0.3125, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 57 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0232, + "frac_reward_zero_std": 0.5, + "grad_norm": 46.34135818481445, + "kl": 0.4858321249485016, + "learning_rate": 1.998806137341434e-06, + "loss": -0.8729, + "num_tokens": 626040.0, + "reward": 0.1875, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.1875, + "rewards/compute_reward/std": 0.40311288833618164, + "step": 58 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0236, + "frac_reward_zero_std": 0.5, + "grad_norm": 50.77784729003906, + "kl": 0.4124316945672035, + "learning_rate": 1.998440764181981e-06, + "loss": 0.4681, + "num_tokens": 634920.0, + "reward": 0.1875, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.1875, + "rewards/compute_reward/std": 0.40311288833618164, + "step": 59 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.024, + "frac_reward_zero_std": 0.25, + "grad_norm": 62.64232635498047, + "kl": 0.4631865471601486, + "learning_rate": 1.9980267284282714e-06, + "loss": -0.0896, + "num_tokens": 643800.0, + "reward": 0.1875, + "reward_std": 0.375, + "rewards/compute_reward/mean": 0.1875, + "rewards/compute_reward/std": 0.40311288833618164, + "step": 60 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0244, + "frac_reward_zero_std": 0.5, + "grad_norm": 55.10979080200195, + "kl": 0.3911132216453552, + "learning_rate": 1.997564050259824e-06, + "loss": 0.1265, + "num_tokens": 652680.0, + "reward": 0.375, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.375, + "rewards/compute_reward/std": 0.5, + "step": 61 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0248, + "frac_reward_zero_std": 0.25, + "grad_norm": 75.01067352294922, + "kl": 0.4750595986843109, + "learning_rate": 1.99705275222692e-06, + "loss": -0.8729, + "num_tokens": 661560.0, + "reward": 0.5625, + "reward_std": 0.375, + "rewards/compute_reward/mean": 0.5625, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 62 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0252, + "frac_reward_zero_std": 0.0, + "grad_norm": 69.7686996459961, + "kl": 0.4622643366456032, + "learning_rate": 1.9964928592495045e-06, + "loss": -0.3061, + "num_tokens": 670440.0, + "reward": 0.4375, + "reward_std": 0.51933753490448, + "rewards/compute_reward/mean": 0.4375, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 63 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0256, + "frac_reward_zero_std": 0.0, + "grad_norm": 78.1102294921875, + "kl": 0.4316819906234741, + "learning_rate": 1.99588439861597e-06, + "loss": 0.8431, + "num_tokens": 679320.0, + "reward": 0.5625, + "reward_std": 0.51933753490448, + "rewards/compute_reward/mean": 0.5625, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 64 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.026, + "frac_reward_zero_std": 0.25, + "grad_norm": 70.63719940185547, + "kl": 0.4158880114555359, + "learning_rate": 1.995227399981831e-06, + "loss": 0.0597, + "num_tokens": 688200.0, + "reward": 0.5625, + "reward_std": 0.41367512941360474, + "rewards/compute_reward/mean": 0.5625, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 65 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0264, + "frac_reward_zero_std": 0.25, + "grad_norm": 56.348411560058594, + "kl": 0.30301499366760254, + "learning_rate": 1.994521895368273e-06, + "loss": -0.2153, + "num_tokens": 697080.0, + "reward": 0.5625, + "reward_std": 0.41367512941360474, + "rewards/compute_reward/mean": 0.5625, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 66 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0268, + "frac_reward_zero_std": 0.5, + "grad_norm": 25.096176147460938, + "kl": 0.27180512621998787, + "learning_rate": 1.9937679191605962e-06, + "loss": 0.7175, + "num_tokens": 705960.0, + "reward": 0.1875, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.1875, + "rewards/compute_reward/std": 0.40311288833618164, + "step": 67 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0272, + "frac_reward_zero_std": 0.25, + "grad_norm": 43.11947250366211, + "kl": 0.27942386642098427, + "learning_rate": 1.992965508106537e-06, + "loss": -0.3403, + "num_tokens": 714840.0, + "reward": 0.6875, + "reward_std": 0.41367512941360474, + "rewards/compute_reward/mean": 0.6875, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 68 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0276, + "frac_reward_zero_std": 0.5, + "grad_norm": 38.26469802856445, + "kl": 0.24121180921792984, + "learning_rate": 1.9921147013144777e-06, + "loss": -0.124, + "num_tokens": 723720.0, + "reward": 0.1875, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.1875, + "rewards/compute_reward/std": 0.40311288833618164, + "step": 69 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.028, + "frac_reward_zero_std": 1.0, + "grad_norm": 16.814804077148438, + "kl": 0.2414722703397274, + "learning_rate": 1.9912155402515414e-06, + "loss": 0.001, + "num_tokens": 732600.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/compute_reward/mean": 0.0, + "rewards/compute_reward/std": 0.0, + "step": 70 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0284, + "frac_reward_zero_std": 0.25, + "grad_norm": 58.598567962646484, + "kl": 0.4162477105855942, + "learning_rate": 1.99026806874157e-06, + "loss": -0.2483, + "num_tokens": 741480.0, + "reward": 0.4375, + "reward_std": 0.41367512941360474, + "rewards/compute_reward/mean": 0.4375, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 71 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0288, + "frac_reward_zero_std": 0.5, + "grad_norm": 56.5173454284668, + "kl": 0.30467043071985245, + "learning_rate": 1.9892723329629885e-06, + "loss": 0.2512, + "num_tokens": 750360.0, + "reward": 0.25, + "reward_std": 0.28867512941360474, + "rewards/compute_reward/mean": 0.25, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 72 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0292, + "frac_reward_zero_std": 0.5, + "grad_norm": 88.95624542236328, + "kl": 0.35381778329610825, + "learning_rate": 1.9882283814465526e-06, + "loss": 0.5593, + "num_tokens": 759240.0, + "reward": 0.5625, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.5625, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 73 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0296, + "frac_reward_zero_std": 0.5, + "grad_norm": 90.53428649902344, + "kl": 0.3509898856282234, + "learning_rate": 1.987136265072988e-06, + "loss": -0.6815, + "num_tokens": 768120.0, + "reward": 0.625, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.625, + "rewards/compute_reward/std": 0.5, + "step": 74 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.03, + "frac_reward_zero_std": 0.5, + "grad_norm": 40.467559814453125, + "kl": 0.27240653708577156, + "learning_rate": 1.985996037070505e-06, + "loss": 0.501, + "num_tokens": 777000.0, + "reward": 0.375, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.375, + "rewards/compute_reward/std": 0.5, + "step": 75 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0304, + "frac_reward_zero_std": 0.75, + "grad_norm": 42.016510009765625, + "kl": 0.3176127299666405, + "learning_rate": 1.984807753012208e-06, + "loss": -0.3737, + "num_tokens": 785880.0, + "reward": 0.3125, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.3125, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 76 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0308, + "frac_reward_zero_std": 0.5, + "grad_norm": 42.70841598510742, + "kl": 0.40913257747888565, + "learning_rate": 1.983571470813386e-06, + "loss": 0.2181, + "num_tokens": 794760.0, + "reward": 0.4375, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.4375, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 77 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0312, + "frac_reward_zero_std": 0.5, + "grad_norm": 48.8704833984375, + "kl": 0.30612240731716156, + "learning_rate": 1.9822872507286887e-06, + "loss": -0.4652, + "num_tokens": 803640.0, + "reward": 0.4375, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.4375, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 78 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0316, + "frac_reward_zero_std": 1.0, + "grad_norm": 10.744608879089355, + "kl": 0.3573462963104248, + "learning_rate": 1.9809551553491913e-06, + "loss": 0.2514, + "num_tokens": 812520.0, + "reward": 0.25, + "reward_std": 0.0, + "rewards/compute_reward/mean": 0.25, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 79 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.032, + "frac_reward_zero_std": 0.25, + "grad_norm": 44.66023254394531, + "kl": 0.2872606888413429, + "learning_rate": 1.979575249599344e-06, + "loss": 0.1596, + "num_tokens": 821400.0, + "reward": 0.625, + "reward_std": 0.39433756470680237, + "rewards/compute_reward/mean": 0.625, + "rewards/compute_reward/std": 0.5, + "step": 80 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0324, + "frac_reward_zero_std": 0.5, + "grad_norm": 48.97246170043945, + "kl": 0.41179851442575455, + "learning_rate": 1.9781476007338054e-06, + "loss": -0.2818, + "num_tokens": 830280.0, + "reward": 0.375, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.375, + "rewards/compute_reward/std": 0.5, + "step": 81 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0328, + "frac_reward_zero_std": 0.75, + "grad_norm": 47.648284912109375, + "kl": 0.361647367477417, + "learning_rate": 1.9766722783341677e-06, + "loss": -0.09, + "num_tokens": 839160.0, + "reward": 0.375, + "reward_std": 0.14433756470680237, + "rewards/compute_reward/mean": 0.375, + "rewards/compute_reward/std": 0.5, + "step": 82 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0332, + "frac_reward_zero_std": 0.5, + "grad_norm": 68.1531982421875, + "kl": 0.35677963495254517, + "learning_rate": 1.975149354305563e-06, + "loss": 0.0929, + "num_tokens": 848040.0, + "reward": 0.4375, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.4375, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 83 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0336, + "frac_reward_zero_std": 1.0, + "grad_norm": 25.728612899780273, + "kl": 0.299656730145216, + "learning_rate": 1.97357890287316e-06, + "loss": 0.1262, + "num_tokens": 856920.0, + "reward": 0.25, + "reward_std": 0.0, + "rewards/compute_reward/mean": 0.25, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 84 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.034, + "frac_reward_zero_std": 0.25, + "grad_norm": 43.74058151245117, + "kl": 0.3780274763703346, + "learning_rate": 1.9719610005785463e-06, + "loss": -0.215, + "num_tokens": 865800.0, + "reward": 0.375, + "reward_std": 0.39433756470680237, + "rewards/compute_reward/mean": 0.375, + "rewards/compute_reward/std": 0.5, + "step": 85 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0344, + "frac_reward_zero_std": 1.0, + "grad_norm": 22.789318084716797, + "kl": 0.361059308052063, + "learning_rate": 1.9702957262759963e-06, + "loss": 0.2179, + "num_tokens": 874680.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/compute_reward/mean": 0.0, + "rewards/compute_reward/std": 0.0, + "step": 86 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0348, + "frac_reward_zero_std": 0.75, + "grad_norm": 35.32533264160156, + "kl": 0.44058068096637726, + "learning_rate": 1.968583161128631e-06, + "loss": 0.0018, + "num_tokens": 883560.0, + "reward": 0.3125, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.3125, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 87 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0352, + "frac_reward_zero_std": 0.5, + "grad_norm": 48.396873474121094, + "kl": 0.40034161880612373, + "learning_rate": 1.9668233886044593e-06, + "loss": 0.3765, + "num_tokens": 892440.0, + "reward": 0.4375, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.4375, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 88 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0356, + "frac_reward_zero_std": 0.5, + "grad_norm": 51.92245101928711, + "kl": 0.582250103354454, + "learning_rate": 1.9650164944723112e-06, + "loss": -0.2811, + "num_tokens": 901320.0, + "reward": 0.4375, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.4375, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 89 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.036, + "frac_reward_zero_std": 0.75, + "grad_norm": 39.04520797729492, + "kl": 0.46102364361286163, + "learning_rate": 1.963162566797658e-06, + "loss": 0.1603, + "num_tokens": 910200.0, + "reward": 0.6875, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.6875, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 90 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0364, + "frac_reward_zero_std": 0.25, + "grad_norm": 64.48860168457031, + "kl": 0.7562894448637962, + "learning_rate": 1.9612616959383188e-06, + "loss": 0.2195, + "num_tokens": 919080.0, + "reward": 0.625, + "reward_std": 0.39433756470680237, + "rewards/compute_reward/mean": 0.625, + "rewards/compute_reward/std": 0.5, + "step": 91 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0368, + "frac_reward_zero_std": 0.5, + "grad_norm": 71.13645935058594, + "kl": 0.584452673792839, + "learning_rate": 1.9593139745400573e-06, + "loss": -0.2141, + "num_tokens": 927960.0, + "reward": 0.625, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.625, + "rewards/compute_reward/std": 0.5, + "step": 92 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0372, + "frac_reward_zero_std": 0.25, + "grad_norm": 44.049346923828125, + "kl": 0.5378477424383163, + "learning_rate": 1.957319497532067e-06, + "loss": -0.2143, + "num_tokens": 936840.0, + "reward": 0.5, + "reward_std": 0.39433756470680237, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 93 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0376, + "frac_reward_zero_std": 0.25, + "grad_norm": 75.5696792602539, + "kl": 0.5544816628098488, + "learning_rate": 1.9552783621223435e-06, + "loss": 0.5601, + "num_tokens": 945720.0, + "reward": 0.75, + "reward_std": 0.39433756470680237, + "rewards/compute_reward/mean": 0.75, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 94 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.038, + "frac_reward_zero_std": 1.0, + "grad_norm": 17.455116271972656, + "kl": 0.3371001221239567, + "learning_rate": 1.953190667792947e-06, + "loss": -0.59, + "num_tokens": 954600.0, + "reward": 0.5, + "reward_std": 0.0, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 95 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0384, + "frac_reward_zero_std": 0.25, + "grad_norm": 59.06554412841797, + "kl": 0.3672589622437954, + "learning_rate": 1.9510565162951534e-06, + "loss": 0.2179, + "num_tokens": 963480.0, + "reward": 0.625, + "reward_std": 0.39433756470680237, + "rewards/compute_reward/mean": 0.625, + "rewards/compute_reward/std": 0.5, + "step": 96 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0388, + "frac_reward_zero_std": 0.5, + "grad_norm": 22.509157180786133, + "kl": 0.18590721115469933, + "learning_rate": 1.9488760116444964e-06, + "loss": -0.3407, + "num_tokens": 972360.0, + "reward": 0.6875, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.6875, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 97 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0392, + "frac_reward_zero_std": 0.5, + "grad_norm": 36.51370620727539, + "kl": 0.25807878002524376, + "learning_rate": 1.9466492601156963e-06, + "loss": 0.126, + "num_tokens": 981240.0, + "reward": 0.25, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.25, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 98 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0396, + "frac_reward_zero_std": 0.75, + "grad_norm": 13.788187026977539, + "kl": 0.23087293654680252, + "learning_rate": 1.944376370237481e-06, + "loss": 0.2509, + "num_tokens": 990120.0, + "reward": 0.6875, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.6875, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 99 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.04, + "frac_reward_zero_std": 0.5, + "grad_norm": 8.3355073928833, + "kl": 0.1271139644086361, + "learning_rate": 1.9420574527872966e-06, + "loss": -0.1245, + "num_tokens": 999000.0, + "reward": 0.375, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.375, + "rewards/compute_reward/std": 0.5, + "step": 100 + }, + { + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.04, + "eval_loss": -0.00047406076919287443, + "eval_runtime": 10044.1197, + "eval_samples_per_second": 0.02, + "eval_steps_per_second": 0.005, + "frac_reward_zero_std": 0.66, + "num_tokens": 1110000.0, + "reward": 0.415, + "reward_std": 0.17928203105926513, + "rewards/compute_reward/mean": 0.415, + "rewards/compute_reward/std": 0.17928203105926513, + "step": 100 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0404, + "frac_reward_zero_std": 0.75, + "grad_norm": 1.7402211427688599, + "kl": 0.12196930311620235, + "learning_rate": 1.9396926207859082e-06, + "loss": 0.217, + "num_tokens": 1118880.0, + "reward": 0.375, + "reward_std": 0.14433756470680237, + "rewards/compute_reward/mean": 0.375, + "rewards/compute_reward/std": 0.5, + "step": 101 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0408, + "frac_reward_zero_std": 0.5, + "grad_norm": 7.048949241638184, + "kl": 0.19601425155997276, + "learning_rate": 1.9372819894918914e-06, + "loss": 0.0008, + "num_tokens": 1127760.0, + "reward": 0.75, + "reward_std": 0.28867512941360474, + "rewards/compute_reward/mean": 0.75, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 102 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0412, + "frac_reward_zero_std": 0.5, + "grad_norm": 14.87850284576416, + "kl": 0.20362638682127, + "learning_rate": 1.9348256763960142e-06, + "loss": -0.2157, + "num_tokens": 1136640.0, + "reward": 0.5, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 103 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0416, + "frac_reward_zero_std": 1.0, + "grad_norm": 19.110157012939453, + "kl": 0.17501263320446014, + "learning_rate": 1.9323238012155122e-06, + "loss": 0.0007, + "num_tokens": 1145520.0, + "reward": 0.5, + "reward_std": 0.0, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 104 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.042, + "frac_reward_zero_std": 0.5, + "grad_norm": 16.843177795410156, + "kl": 0.2675781920552254, + "learning_rate": 1.929776485888251e-06, + "loss": 0.376, + "num_tokens": 1154400.0, + "reward": 0.375, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.375, + "rewards/compute_reward/std": 0.5, + "step": 105 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0424, + "frac_reward_zero_std": 0.75, + "grad_norm": 24.672325134277344, + "kl": 0.2085116021335125, + "learning_rate": 1.9271838545667875e-06, + "loss": -0.3741, + "num_tokens": 1163280.0, + "reward": 0.625, + "reward_std": 0.14433756470680237, + "rewards/compute_reward/mean": 0.625, + "rewards/compute_reward/std": 0.5, + "step": 106 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0428, + "frac_reward_zero_std": 0.75, + "grad_norm": 3.470179319381714, + "kl": 0.3066542334854603, + "learning_rate": 1.9245460336123133e-06, + "loss": 0.1262, + "num_tokens": 1172160.0, + "reward": 0.0625, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.0625, + "rewards/compute_reward/std": 0.25, + "step": 107 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0432, + "frac_reward_zero_std": 0.5, + "grad_norm": 33.54941940307617, + "kl": 0.36576371639966965, + "learning_rate": 1.9218631515885003e-06, + "loss": 0.1264, + "num_tokens": 1181040.0, + "reward": 0.25, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.25, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 108 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0436, + "frac_reward_zero_std": 0.75, + "grad_norm": 92.37312316894531, + "kl": 1.9885881915688515, + "learning_rate": 1.9191353392552343e-06, + "loss": 0.008, + "num_tokens": 1189920.0, + "reward": 0.3125, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.3125, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 109 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.044, + "frac_reward_zero_std": 0.75, + "grad_norm": 33.686317443847656, + "kl": 0.45819035917520523, + "learning_rate": 1.9163627295622395e-06, + "loss": -0.3731, + "num_tokens": 1198800.0, + "reward": 0.0625, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.0625, + "rewards/compute_reward/std": 0.25, + "step": 110 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0444, + "frac_reward_zero_std": 0.5, + "grad_norm": 25.443355560302734, + "kl": 0.4073995351791382, + "learning_rate": 1.9135454576426007e-06, + "loss": -0.4983, + "num_tokens": 1207680.0, + "reward": 0.625, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.625, + "rewards/compute_reward/std": 0.5, + "step": 111 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0448, + "frac_reward_zero_std": 0.75, + "grad_norm": 48.13676834106445, + "kl": 0.4856240525841713, + "learning_rate": 1.910683660806177e-06, + "loss": 0.4103, + "num_tokens": 1216560.0, + "reward": 0.375, + "reward_std": 0.14433756470680237, + "rewards/compute_reward/mean": 0.375, + "rewards/compute_reward/std": 0.5, + "step": 112 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0452, + "frac_reward_zero_std": 0.5, + "grad_norm": 26.295862197875977, + "kl": 0.3765817731618881, + "learning_rate": 1.9077774785329087e-06, + "loss": 0.5929, + "num_tokens": 1225440.0, + "reward": 0.5, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 113 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0456, + "frac_reward_zero_std": 0.5, + "grad_norm": 60.09946823120117, + "kl": 0.5815477445721626, + "learning_rate": 1.9048270524660196e-06, + "loss": -0.5891, + "num_tokens": 1234320.0, + "reward": 0.25, + "reward_std": 0.28867512941360474, + "rewards/compute_reward/mean": 0.25, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 114 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.046, + "frac_reward_zero_std": 0.75, + "grad_norm": 43.9576530456543, + "kl": 0.48454301804304123, + "learning_rate": 1.9018325264051138e-06, + "loss": 0.4349, + "num_tokens": 1243200.0, + "reward": 0.625, + "reward_std": 0.14433756470680237, + "rewards/compute_reward/mean": 0.625, + "rewards/compute_reward/std": 0.5, + "step": 115 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0464, + "frac_reward_zero_std": 0.5, + "grad_norm": 34.4670295715332, + "kl": 0.6097010597586632, + "learning_rate": 1.8987940462991669e-06, + "loss": 0.4689, + "num_tokens": 1252080.0, + "reward": 0.6875, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.6875, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 116 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0468, + "frac_reward_zero_std": 0.75, + "grad_norm": 44.33097839355469, + "kl": 0.6179208308458328, + "learning_rate": 1.8957117602394128e-06, + "loss": -0.8969, + "num_tokens": 1260960.0, + "reward": 0.375, + "reward_std": 0.14433756470680237, + "rewards/compute_reward/mean": 0.375, + "rewards/compute_reward/std": 0.5, + "step": 117 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0472, + "frac_reward_zero_std": 0.5, + "grad_norm": 36.61655044555664, + "kl": 0.6962894722819328, + "learning_rate": 1.8925858184521255e-06, + "loss": -0.3721, + "num_tokens": 1269840.0, + "reward": 0.5625, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.5625, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 118 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0476, + "frac_reward_zero_std": 0.25, + "grad_norm": 80.13101959228516, + "kl": 0.6878734081983566, + "learning_rate": 1.8894163732912974e-06, + "loss": 0.3442, + "num_tokens": 1278720.0, + "reward": 0.3125, + "reward_std": 0.375, + "rewards/compute_reward/mean": 0.3125, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 119 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.048, + "frac_reward_zero_std": 0.75, + "grad_norm": 90.39923858642578, + "kl": 0.7930669263005257, + "learning_rate": 1.8862035792312146e-06, + "loss": 0.1281, + "num_tokens": 1287600.0, + "reward": 0.4375, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.4375, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 120 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0484, + "frac_reward_zero_std": 1.0, + "grad_norm": 37.05628204345703, + "kl": 0.7748868465423584, + "learning_rate": 1.8829475928589268e-06, + "loss": 0.1281, + "num_tokens": 1296480.0, + "reward": 0.5, + "reward_std": 0.0, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 121 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0488, + "frac_reward_zero_std": 0.0, + "grad_norm": 85.06033325195312, + "kl": 0.9438418298959732, + "learning_rate": 1.8796485728666165e-06, + "loss": -0.6791, + "num_tokens": 1305360.0, + "reward": 0.4375, + "reward_std": 0.51933753490448, + "rewards/compute_reward/mean": 0.4375, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 122 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0492, + "frac_reward_zero_std": 0.5, + "grad_norm": 116.63643646240234, + "kl": 1.1643976122140884, + "learning_rate": 1.8763066800438634e-06, + "loss": 0.9375, + "num_tokens": 1314240.0, + "reward": 0.5, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 123 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0496, + "frac_reward_zero_std": 0.75, + "grad_norm": 87.37930297851562, + "kl": 1.1998874247074127, + "learning_rate": 1.8729220772698095e-06, + "loss": -0.1202, + "num_tokens": 1323120.0, + "reward": 0.1875, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.1875, + "rewards/compute_reward/std": 0.40311288833618164, + "step": 124 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.05, + "frac_reward_zero_std": 0.75, + "grad_norm": 78.85758972167969, + "kl": 1.4393538534641266, + "learning_rate": 1.869494929505219e-06, + "loss": 0.0058, + "num_tokens": 1332000.0, + "reward": 0.1875, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.1875, + "rewards/compute_reward/std": 0.40311288833618164, + "step": 125 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0504, + "frac_reward_zero_std": 0.25, + "grad_norm": 103.32394409179688, + "kl": 2.1740673780441284, + "learning_rate": 1.8660254037844386e-06, + "loss": 0.1337, + "num_tokens": 1340880.0, + "reward": 0.5625, + "reward_std": 0.375, + "rewards/compute_reward/mean": 0.5625, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 126 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0508, + "frac_reward_zero_std": 0.25, + "grad_norm": 121.61659240722656, + "kl": 1.3770731836557388, + "learning_rate": 1.8625136692072574e-06, + "loss": 0.097, + "num_tokens": 1349760.0, + "reward": 0.25, + "reward_std": 0.39433756470680237, + "rewards/compute_reward/mean": 0.25, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 127 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0512, + "frac_reward_zero_std": 0.75, + "grad_norm": 87.8947525024414, + "kl": 1.433599203824997, + "learning_rate": 1.8589598969306644e-06, + "loss": 0.0972, + "num_tokens": 1358640.0, + "reward": 0.125, + "reward_std": 0.14433756470680237, + "rewards/compute_reward/mean": 0.125, + "rewards/compute_reward/std": 0.3415650427341461, + "step": 128 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0516, + "frac_reward_zero_std": 0.5, + "grad_norm": 99.49671936035156, + "kl": 1.3952637314796448, + "learning_rate": 1.8553642601605066e-06, + "loss": -0.0524, + "num_tokens": 1367520.0, + "reward": 0.375, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.375, + "rewards/compute_reward/std": 0.5, + "step": 129 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.052, + "frac_reward_zero_std": 0.75, + "grad_norm": 44.72431564331055, + "kl": 1.3837022185325623, + "learning_rate": 1.8517269341430474e-06, + "loss": -0.6193, + "num_tokens": 1376400.0, + "reward": 0.4375, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.4375, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 130 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0524, + "frac_reward_zero_std": 0.5, + "grad_norm": 72.39948272705078, + "kl": 1.6086200773715973, + "learning_rate": 1.8480480961564257e-06, + "loss": 0.3479, + "num_tokens": 1385280.0, + "reward": 0.6875, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.6875, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 131 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0528, + "frac_reward_zero_std": 0.75, + "grad_norm": 75.30799102783203, + "kl": 1.511486679315567, + "learning_rate": 1.844327925502015e-06, + "loss": 0.0395, + "num_tokens": 1394160.0, + "reward": 0.0625, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.0625, + "rewards/compute_reward/std": 0.25, + "step": 132 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0532, + "frac_reward_zero_std": 0.75, + "grad_norm": 67.3117446899414, + "kl": 1.4110450446605682, + "learning_rate": 1.8405666034956842e-06, + "loss": -0.5523, + "num_tokens": 1403040.0, + "reward": 0.125, + "reward_std": 0.14433756470680237, + "rewards/compute_reward/mean": 0.125, + "rewards/compute_reward/std": 0.3415650427341461, + "step": 133 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0536, + "frac_reward_zero_std": 0.25, + "grad_norm": 91.60472869873047, + "kl": 1.495105355978012, + "learning_rate": 1.8367643134589616e-06, + "loss": 0.2224, + "num_tokens": 1411920.0, + "reward": 0.5, + "reward_std": 0.39433756470680237, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 134 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.054, + "frac_reward_zero_std": 0.75, + "grad_norm": 89.93647766113281, + "kl": 1.5621980130672455, + "learning_rate": 1.8329212407100993e-06, + "loss": 0.2227, + "num_tokens": 1420800.0, + "reward": 0.375, + "reward_std": 0.14433756470680237, + "rewards/compute_reward/mean": 0.375, + "rewards/compute_reward/std": 0.5, + "step": 135 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0544, + "frac_reward_zero_std": 1.0, + "grad_norm": 38.31406784057617, + "kl": 1.4404120743274689, + "learning_rate": 1.8290375725550415e-06, + "loss": 0.0058, + "num_tokens": 1429680.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/compute_reward/mean": 0.0, + "rewards/compute_reward/std": 0.0, + "step": 136 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0548, + "frac_reward_zero_std": 0.75, + "grad_norm": 67.23009490966797, + "kl": 1.8493570387363434, + "learning_rate": 1.825113498278295e-06, + "loss": 0.2239, + "num_tokens": 1438560.0, + "reward": 0.125, + "reward_std": 0.14433756470680237, + "rewards/compute_reward/mean": 0.125, + "rewards/compute_reward/std": 0.3415650427341461, + "step": 137 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0552, + "frac_reward_zero_std": 1.0, + "grad_norm": 26.11929702758789, + "kl": 1.655295968055725, + "learning_rate": 1.821149209133704e-06, + "loss": -0.2098, + "num_tokens": 1447440.0, + "reward": 0.25, + "reward_std": 0.0, + "rewards/compute_reward/mean": 0.25, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 138 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0556, + "frac_reward_zero_std": 0.75, + "grad_norm": 68.21998596191406, + "kl": 1.6474502980709076, + "learning_rate": 1.8171448983351283e-06, + "loss": -0.1184, + "num_tokens": 1456320.0, + "reward": 0.3125, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.3125, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 139 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.056, + "frac_reward_zero_std": 0.5, + "grad_norm": 74.41326141357422, + "kl": 1.287375420331955, + "learning_rate": 1.8131007610470274e-06, + "loss": 0.0051, + "num_tokens": 1465200.0, + "reward": 0.5, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 140 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0564, + "frac_reward_zero_std": 0.25, + "grad_norm": 69.33372497558594, + "kl": 1.3127607554197311, + "learning_rate": 1.8090169943749474e-06, + "loss": 0.8801, + "num_tokens": 1474080.0, + "reward": 0.5, + "reward_std": 0.39433756470680237, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 141 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0568, + "frac_reward_zero_std": 0.25, + "grad_norm": 76.3515625, + "kl": 1.296535700559616, + "learning_rate": 1.804893797355914e-06, + "loss": -0.8027, + "num_tokens": 1482960.0, + "reward": 0.375, + "reward_std": 0.39433756470680237, + "rewards/compute_reward/mean": 0.375, + "rewards/compute_reward/std": 0.5, + "step": 142 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0572, + "frac_reward_zero_std": 0.75, + "grad_norm": 87.84119415283203, + "kl": 1.1495089530944824, + "learning_rate": 1.8007313709487333e-06, + "loss": -0.1873, + "num_tokens": 1491840.0, + "reward": 0.0625, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.0625, + "rewards/compute_reward/std": 0.25, + "step": 143 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0576, + "frac_reward_zero_std": 0.25, + "grad_norm": 45.25433349609375, + "kl": 1.1036857962608337, + "learning_rate": 1.7965299180241961e-06, + "loss": 0.2209, + "num_tokens": 1500720.0, + "reward": 0.5625, + "reward_std": 0.41367512941360474, + "rewards/compute_reward/mean": 0.5625, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 144 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.058, + "frac_reward_zero_std": 0.5, + "grad_norm": 48.654361724853516, + "kl": 1.1962954103946686, + "learning_rate": 1.7922896433551906e-06, + "loss": 0.5382, + "num_tokens": 1509600.0, + "reward": 0.125, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.125, + "rewards/compute_reward/std": 0.3415650427341461, + "step": 145 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0584, + "frac_reward_zero_std": 0.75, + "grad_norm": 56.95619201660156, + "kl": 1.3726016283035278, + "learning_rate": 1.7880107536067217e-06, + "loss": -0.2445, + "num_tokens": 1518480.0, + "reward": 0.3125, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.3125, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 146 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0588, + "frac_reward_zero_std": 0.75, + "grad_norm": 81.80111694335938, + "kl": 1.270168662071228, + "learning_rate": 1.7836934573258397e-06, + "loss": 0.0051, + "num_tokens": 1527360.0, + "reward": 0.4375, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.4375, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 147 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0592, + "frac_reward_zero_std": 0.75, + "grad_norm": 55.87933349609375, + "kl": 1.5507261157035828, + "learning_rate": 1.7793379649314742e-06, + "loss": -0.4602, + "num_tokens": 1536240.0, + "reward": 0.625, + "reward_std": 0.14433756470680237, + "rewards/compute_reward/mean": 0.625, + "rewards/compute_reward/std": 0.5, + "step": 148 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0596, + "frac_reward_zero_std": 0.5, + "grad_norm": 40.03895950317383, + "kl": 1.3443613648414612, + "learning_rate": 1.7749444887041795e-06, + "loss": -0.2446, + "num_tokens": 1545120.0, + "reward": 0.1875, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.1875, + "rewards/compute_reward/std": 0.40311288833618164, + "step": 149 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.06, + "frac_reward_zero_std": 0.0, + "grad_norm": 60.483856201171875, + "kl": 1.244696855545044, + "learning_rate": 1.7705132427757892e-06, + "loss": 0.2884, + "num_tokens": 1554000.0, + "reward": 0.4375, + "reward_std": 0.51933753490448, + "rewards/compute_reward/mean": 0.4375, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 150 + }, + { + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.06, + "eval_loss": -0.0004801125905942172, + "eval_runtime": 5038.6351, + "eval_samples_per_second": 0.04, + "eval_steps_per_second": 0.01, + "frac_reward_zero_std": 0.64, + "num_tokens": 1665000.0, + "reward": 0.37, + "reward_std": 0.18928203105926514, + "rewards/compute_reward/mean": 0.37, + "rewards/compute_reward/std": 0.18928203105926514, + "step": 150 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0604, + "frac_reward_zero_std": 1.0, + "grad_norm": 102.57103729248047, + "kl": 0.8290184140205383, + "learning_rate": 1.766044443118978e-06, + "loss": 0.5032, + "num_tokens": 1673880.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/compute_reward/mean": 0.0, + "rewards/compute_reward/std": 0.0, + "step": 151 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0608, + "frac_reward_zero_std": 0.75, + "grad_norm": 10.56777286529541, + "kl": 1.1542262136936188, + "learning_rate": 1.7615383075367368e-06, + "loss": 0.2211, + "num_tokens": 1682760.0, + "reward": 0.125, + "reward_std": 0.14433756470680237, + "rewards/compute_reward/mean": 0.125, + "rewards/compute_reward/std": 0.3415650427341461, + "step": 152 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0612, + "frac_reward_zero_std": 0.5, + "grad_norm": 37.89947509765625, + "kl": 0.9573570787906647, + "learning_rate": 1.7569950556517563e-06, + "loss": 0.1623, + "num_tokens": 1691640.0, + "reward": 0.375, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.375, + "rewards/compute_reward/std": 0.5, + "step": 153 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0616, + "frac_reward_zero_std": 1.0, + "grad_norm": 30.090473175048828, + "kl": 0.8319032788276672, + "learning_rate": 1.7524149088957242e-06, + "loss": -0.3716, + "num_tokens": 1700520.0, + "reward": 0.5, + "reward_std": 0.0, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 154 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.062, + "frac_reward_zero_std": 0.5, + "grad_norm": 37.990379333496094, + "kl": 0.7478515356779099, + "learning_rate": 1.747798090498532e-06, + "loss": 0.0365, + "num_tokens": 1709400.0, + "reward": 0.5625, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.5625, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 155 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0624, + "frac_reward_zero_std": 0.5, + "grad_norm": 27.682296752929688, + "kl": 0.902895525097847, + "learning_rate": 1.743144825477394e-06, + "loss": 0.528, + "num_tokens": 1718280.0, + "reward": 0.1875, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.1875, + "rewards/compute_reward/std": 0.40311288833618164, + "step": 156 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0628, + "frac_reward_zero_std": 0.75, + "grad_norm": 50.9190788269043, + "kl": 0.8830957859754562, + "learning_rate": 1.738455340625884e-06, + "loss": -0.4294, + "num_tokens": 1727160.0, + "reward": 0.3125, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.3125, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 157 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0632, + "frac_reward_zero_std": 0.5, + "grad_norm": 20.148906707763672, + "kl": 0.7896845638751984, + "learning_rate": 1.7337298645028762e-06, + "loss": -0.4967, + "num_tokens": 1736040.0, + "reward": 0.625, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.625, + "rewards/compute_reward/std": 0.5, + "step": 158 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0636, + "frac_reward_zero_std": 0.5, + "grad_norm": 32.239356994628906, + "kl": 0.804187685251236, + "learning_rate": 1.7289686274214115e-06, + "loss": 0.7196, + "num_tokens": 1744920.0, + "reward": 0.4375, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.4375, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 159 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.064, + "frac_reward_zero_std": 0.5, + "grad_norm": 24.03925895690918, + "kl": 0.8029276132583618, + "learning_rate": 1.7241718614374676e-06, + "loss": -0.5882, + "num_tokens": 1753800.0, + "reward": 0.3125, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.3125, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 160 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0644, + "frac_reward_zero_std": 0.75, + "grad_norm": 30.868242263793945, + "kl": 0.8987027704715729, + "learning_rate": 1.719339800338651e-06, + "loss": 0.2535, + "num_tokens": 1762680.0, + "reward": 0.4375, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.4375, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 161 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0648, + "frac_reward_zero_std": 0.5, + "grad_norm": 19.09716033935547, + "kl": 0.9851841777563095, + "learning_rate": 1.7144726796328032e-06, + "loss": -0.2125, + "num_tokens": 1771560.0, + "reward": 0.1875, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.1875, + "rewards/compute_reward/std": 0.40311288833618164, + "step": 162 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0652, + "frac_reward_zero_std": 0.5, + "grad_norm": 32.937808990478516, + "kl": 0.7938939332962036, + "learning_rate": 1.7095707365365209e-06, + "loss": 0.4696, + "num_tokens": 1780440.0, + "reward": 0.125, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.125, + "rewards/compute_reward/std": 0.3415650427341461, + "step": 163 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0656, + "frac_reward_zero_std": 0.25, + "grad_norm": 37.37956619262695, + "kl": 0.7686180472373962, + "learning_rate": 1.7046342099635947e-06, + "loss": 0.311, + "num_tokens": 1789320.0, + "reward": 0.3125, + "reward_std": 0.41367512941360474, + "rewards/compute_reward/mean": 0.3125, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 164 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.066, + "frac_reward_zero_std": 0.75, + "grad_norm": 29.587915420532227, + "kl": 0.6731294244527817, + "learning_rate": 1.6996633405133653e-06, + "loss": -0.5552, + "num_tokens": 1798200.0, + "reward": 0.6875, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.6875, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 165 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0664, + "frac_reward_zero_std": 1.0, + "grad_norm": 28.84545135498047, + "kl": 0.874516099691391, + "learning_rate": 1.6946583704589972e-06, + "loss": 0.0035, + "num_tokens": 1807080.0, + "reward": 0.75, + "reward_std": 0.0, + "rewards/compute_reward/mean": 0.75, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 166 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0668, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.2955371141433716, + "kl": 0.7283145040273666, + "learning_rate": 1.6896195437356697e-06, + "loss": 0.0029, + "num_tokens": 1815960.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/compute_reward/mean": 0.0, + "rewards/compute_reward/std": 0.0, + "step": 167 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0672, + "frac_reward_zero_std": 0.5, + "grad_norm": 17.424720764160156, + "kl": 0.9037010669708252, + "learning_rate": 1.6845471059286886e-06, + "loss": 0.2536, + "num_tokens": 1824840.0, + "reward": 0.25, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.25, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 168 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0676, + "frac_reward_zero_std": 0.75, + "grad_norm": 26.717079162597656, + "kl": 0.6133365780115128, + "learning_rate": 1.6794413042615166e-06, + "loss": -0.031, + "num_tokens": 1833720.0, + "reward": 0.375, + "reward_std": 0.14433756470680237, + "rewards/compute_reward/mean": 0.375, + "rewards/compute_reward/std": 0.5, + "step": 169 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.068, + "frac_reward_zero_std": 0.75, + "grad_norm": 24.280805587768555, + "kl": 0.7366704791784286, + "learning_rate": 1.6743023875837233e-06, + "loss": -0.0885, + "num_tokens": 1842600.0, + "reward": 0.3125, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.3125, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 170 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0684, + "frac_reward_zero_std": 0.5, + "grad_norm": 23.280994415283203, + "kl": 0.752781942486763, + "learning_rate": 1.669130606358858e-06, + "loss": -0.3719, + "num_tokens": 1851480.0, + "reward": 0.25, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.25, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 171 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0688, + "frac_reward_zero_std": 1.0, + "grad_norm": 28.28022575378418, + "kl": 0.7437139600515366, + "learning_rate": 1.6639262126522415e-06, + "loss": 0.2529, + "num_tokens": 1860360.0, + "reward": 0.5, + "reward_std": 0.0, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 172 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0692, + "frac_reward_zero_std": 0.25, + "grad_norm": 18.955217361450195, + "kl": 0.51456718146801, + "learning_rate": 1.6586894601186803e-06, + "loss": 0.0355, + "num_tokens": 1869240.0, + "reward": 0.5, + "reward_std": 0.39433756470680237, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 173 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0696, + "frac_reward_zero_std": 0.75, + "grad_norm": 27.11832618713379, + "kl": 0.6736243739724159, + "learning_rate": 1.6534206039901055e-06, + "loss": -0.0308, + "num_tokens": 1878120.0, + "reward": 0.5625, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.5625, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 174 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.07, + "frac_reward_zero_std": 0.25, + "grad_norm": 31.381895065307617, + "kl": 0.6929000839591026, + "learning_rate": 1.6481199010631309e-06, + "loss": 0.0028, + "num_tokens": 1887000.0, + "reward": 0.4375, + "reward_std": 0.41367512941360474, + "rewards/compute_reward/mean": 0.4375, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 175 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0704, + "frac_reward_zero_std": 1.0, + "grad_norm": 48.79582977294922, + "kl": 0.562452644109726, + "learning_rate": 1.6427876096865393e-06, + "loss": 0.0022, + "num_tokens": 1895880.0, + "reward": 0.25, + "reward_std": 0.0, + "rewards/compute_reward/mean": 0.25, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 176 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0708, + "frac_reward_zero_std": 0.25, + "grad_norm": 14.692187309265137, + "kl": 0.4838453531265259, + "learning_rate": 1.6374239897486897e-06, + "loss": -0.123, + "num_tokens": 1904760.0, + "reward": 0.625, + "reward_std": 0.39433756470680237, + "rewards/compute_reward/mean": 0.625, + "rewards/compute_reward/std": 0.5, + "step": 177 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0712, + "frac_reward_zero_std": 0.5, + "grad_norm": 32.34980773925781, + "kl": 0.5593864694237709, + "learning_rate": 1.6320293026648508e-06, + "loss": 0.0357, + "num_tokens": 1913640.0, + "reward": 0.4375, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.4375, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 178 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0716, + "frac_reward_zero_std": 0.25, + "grad_norm": 47.210731506347656, + "kl": 1.054399996995926, + "learning_rate": 1.6266038113644605e-06, + "loss": -0.3372, + "num_tokens": 1922520.0, + "reward": 0.3125, + "reward_std": 0.41367512941360474, + "rewards/compute_reward/mean": 0.3125, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 179 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.072, + "frac_reward_zero_std": 1.0, + "grad_norm": 25.16176986694336, + "kl": 0.6305755823850632, + "learning_rate": 1.6211477802783102e-06, + "loss": 0.4355, + "num_tokens": 1931400.0, + "reward": 0.5, + "reward_std": 0.0, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 180 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0724, + "frac_reward_zero_std": 1.0, + "grad_norm": 2.085853099822998, + "kl": 0.6664913669228554, + "learning_rate": 1.615661475325658e-06, + "loss": 0.0027, + "num_tokens": 1940280.0, + "reward": 0.25, + "reward_std": 0.0, + "rewards/compute_reward/mean": 0.25, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 181 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0728, + "frac_reward_zero_std": 0.75, + "grad_norm": 0.25921252369880676, + "kl": 0.5506083369255066, + "learning_rate": 1.6101451639012677e-06, + "loss": 0.0022, + "num_tokens": 1949160.0, + "reward": 0.6875, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.6875, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 182 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0732, + "frac_reward_zero_std": 0.75, + "grad_norm": 19.31712532043457, + "kl": 0.5009290054440498, + "learning_rate": 1.604599114862375e-06, + "loss": 0.4349, + "num_tokens": 1958040.0, + "reward": 0.375, + "reward_std": 0.14433756470680237, + "rewards/compute_reward/mean": 0.375, + "rewards/compute_reward/std": 0.5, + "step": 183 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0736, + "frac_reward_zero_std": 1.0, + "grad_norm": 14.426288604736328, + "kl": 0.8416369184851646, + "learning_rate": 1.5990235985155857e-06, + "loss": -0.4296, + "num_tokens": 1966920.0, + "reward": 0.25, + "reward_std": 0.0, + "rewards/compute_reward/mean": 0.25, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 184 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.074, + "frac_reward_zero_std": 0.25, + "grad_norm": 9.469975471496582, + "kl": 0.6057900637388229, + "learning_rate": 1.5934188866037015e-06, + "loss": -0.0311, + "num_tokens": 1975800.0, + "reward": 0.5, + "reward_std": 0.39433756470680237, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 185 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0744, + "frac_reward_zero_std": 0.75, + "grad_norm": 28.550851821899414, + "kl": 0.6094760373234749, + "learning_rate": 1.587785252292473e-06, + "loss": 0.0359, + "num_tokens": 1984680.0, + "reward": 0.625, + "reward_std": 0.14433756470680237, + "rewards/compute_reward/mean": 0.625, + "rewards/compute_reward/std": 0.5, + "step": 186 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0748, + "frac_reward_zero_std": 0.0, + "grad_norm": 14.759550094604492, + "kl": 0.5186473429203033, + "learning_rate": 1.5821229701572893e-06, + "loss": -0.2479, + "num_tokens": 1993560.0, + "reward": 0.5625, + "reward_std": 0.51933753490448, + "rewards/compute_reward/mean": 0.5625, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 187 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0752, + "frac_reward_zero_std": 0.75, + "grad_norm": 24.977191925048828, + "kl": 0.7406383752822876, + "learning_rate": 1.5764323161697932e-06, + "loss": 0.0364, + "num_tokens": 2002440.0, + "reward": 0.125, + "reward_std": 0.14433756470680237, + "rewards/compute_reward/mean": 0.125, + "rewards/compute_reward/std": 0.3415650427341461, + "step": 188 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0756, + "frac_reward_zero_std": 0.5, + "grad_norm": 19.971651077270508, + "kl": 0.578004002571106, + "learning_rate": 1.5707135676844319e-06, + "loss": 0.0938, + "num_tokens": 2011320.0, + "reward": 0.5, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 189 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.076, + "frac_reward_zero_std": 0.5, + "grad_norm": 12.372550964355469, + "kl": 0.4948463439941406, + "learning_rate": 1.564967003424938e-06, + "loss": 0.3099, + "num_tokens": 2020200.0, + "reward": 0.3125, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.3125, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 190 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0764, + "frac_reward_zero_std": 0.5, + "grad_norm": 13.063384056091309, + "kl": 0.4803493767976761, + "learning_rate": 1.5591929034707466e-06, + "loss": 0.0689, + "num_tokens": 2029080.0, + "reward": 0.5, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 191 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0768, + "frac_reward_zero_std": 0.25, + "grad_norm": 20.98461151123047, + "kl": 0.5075587034225464, + "learning_rate": 1.553391549243344e-06, + "loss": -0.4979, + "num_tokens": 2037960.0, + "reward": 0.5625, + "reward_std": 0.375, + "rewards/compute_reward/mean": 0.5625, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 192 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0772, + "frac_reward_zero_std": 0.75, + "grad_norm": 22.815135955810547, + "kl": 0.5010422468185425, + "learning_rate": 1.5475632234925502e-06, + "loss": 0.3769, + "num_tokens": 2046840.0, + "reward": 0.8125, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.8125, + "rewards/compute_reward/std": 0.40311288833618164, + "step": 193 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0776, + "frac_reward_zero_std": 1.0, + "grad_norm": 16.002519607543945, + "kl": 0.4861938878893852, + "learning_rate": 1.54170821028274e-06, + "loss": -0.123, + "num_tokens": 2055720.0, + "reward": 0.25, + "reward_std": 0.0, + "rewards/compute_reward/mean": 0.25, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 194 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.078, + "frac_reward_zero_std": 0.5, + "grad_norm": 12.706498146057129, + "kl": 0.45841076225042343, + "learning_rate": 1.5358267949789964e-06, + "loss": 0.2518, + "num_tokens": 2064600.0, + "reward": 0.25, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.25, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 195 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0784, + "frac_reward_zero_std": 0.25, + "grad_norm": 49.78022003173828, + "kl": 0.5610195696353912, + "learning_rate": 1.5299192642332049e-06, + "loss": 0.0357, + "num_tokens": 2073480.0, + "reward": 0.625, + "reward_std": 0.39433756470680237, + "rewards/compute_reward/mean": 0.625, + "rewards/compute_reward/std": 0.5, + "step": 196 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0788, + "frac_reward_zero_std": 0.25, + "grad_norm": 26.672563552856445, + "kl": 0.4388924762606621, + "learning_rate": 1.5239859059700793e-06, + "loss": -0.2817, + "num_tokens": 2082360.0, + "reward": 0.25, + "reward_std": 0.39433756470680237, + "rewards/compute_reward/mean": 0.25, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 197 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0792, + "frac_reward_zero_std": 1.0, + "grad_norm": 22.82637596130371, + "kl": 0.39291463047266006, + "learning_rate": 1.5180270093731302e-06, + "loss": 0.0016, + "num_tokens": 2091240.0, + "reward": 0.75, + "reward_std": 0.0, + "rewards/compute_reward/mean": 0.75, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 198 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0796, + "frac_reward_zero_std": 0.75, + "grad_norm": 1.730869174003601, + "kl": 0.4376737102866173, + "learning_rate": 1.5120428648705715e-06, + "loss": -0.1232, + "num_tokens": 2100120.0, + "reward": 0.4375, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.4375, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 199 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.08, + "frac_reward_zero_std": 1.0, + "grad_norm": 6.7394537925720215, + "kl": 0.4456604868173599, + "learning_rate": 1.5060337641211636e-06, + "loss": 0.1268, + "num_tokens": 2109000.0, + "reward": 0.5, + "reward_std": 0.0, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 200 + }, + { + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.08, + "eval_loss": 0.0004982490791007876, + "eval_runtime": 7044.2588, + "eval_samples_per_second": 0.028, + "eval_steps_per_second": 0.007, + "frac_reward_zero_std": 0.6122448979591837, + "num_tokens": 2217780.0, + "reward": 0.413265306122449, + "reward_std": 0.19703470444192692, + "rewards/compute_reward/mean": 0.413265306122449, + "rewards/compute_reward/std": 0.19703470444192692, + "step": 200 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0804, + "frac_reward_zero_std": 0.5, + "grad_norm": 16.79532814025879, + "kl": 0.47658685594797134, + "learning_rate": 1.5e-06, + "loss": 0.0019, + "num_tokens": 2226660.0, + "reward": 0.6875, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.6875, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 201 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0808, + "frac_reward_zero_std": 0.5, + "grad_norm": 19.146230697631836, + "kl": 0.47041744738817215, + "learning_rate": 1.4939418665842309e-06, + "loss": 0.0019, + "num_tokens": 2235540.0, + "reward": 0.3125, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.3125, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 202 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0812, + "frac_reward_zero_std": 0.5, + "grad_norm": 10.93484115600586, + "kl": 0.48019571602344513, + "learning_rate": 1.4878596591387326e-06, + "loss": 0.0019, + "num_tokens": 2244420.0, + "reward": 0.5, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 203 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0816, + "frac_reward_zero_std": 0.25, + "grad_norm": 24.61666488647461, + "kl": 0.6040880158543587, + "learning_rate": 1.4817536741017151e-06, + "loss": 0.0024, + "num_tokens": 2253300.0, + "reward": 0.6875, + "reward_std": 0.375, + "rewards/compute_reward/mean": 0.6875, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 204 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.082, + "frac_reward_zero_std": 0.5, + "grad_norm": 21.1997013092041, + "kl": 0.7307578474283218, + "learning_rate": 1.4756242090702753e-06, + "loss": 0.0029, + "num_tokens": 2262180.0, + "reward": 0.375, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.375, + "rewards/compute_reward/std": 0.5, + "step": 205 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0824, + "frac_reward_zero_std": 0.5, + "grad_norm": 21.705259323120117, + "kl": 1.0300785303115845, + "learning_rate": 1.4694715627858908e-06, + "loss": 0.0041, + "num_tokens": 2271060.0, + "reward": 0.125, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.125, + "rewards/compute_reward/std": 0.3415650427341461, + "step": 206 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0828, + "frac_reward_zero_std": 0.75, + "grad_norm": 20.124446868896484, + "kl": 0.7309058308601379, + "learning_rate": 1.4632960351198617e-06, + "loss": 0.0029, + "num_tokens": 2279940.0, + "reward": 0.375, + "reward_std": 0.14433756470680237, + "rewards/compute_reward/mean": 0.375, + "rewards/compute_reward/std": 0.5, + "step": 207 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0832, + "frac_reward_zero_std": 0.75, + "grad_norm": 24.016128540039062, + "kl": 0.8935306221246719, + "learning_rate": 1.4570979270586943e-06, + "loss": 0.0036, + "num_tokens": 2288820.0, + "reward": 0.625, + "reward_std": 0.14433756470680237, + "rewards/compute_reward/mean": 0.625, + "rewards/compute_reward/std": 0.5, + "step": 208 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0836, + "frac_reward_zero_std": 0.75, + "grad_norm": 14.154572486877441, + "kl": 0.9832312911748886, + "learning_rate": 1.4508775406894306e-06, + "loss": 0.0039, + "num_tokens": 2297700.0, + "reward": 0.4375, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.4375, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 209 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.084, + "frac_reward_zero_std": 0.5, + "grad_norm": 27.507604598999023, + "kl": 0.9729635268449783, + "learning_rate": 1.4446351791849273e-06, + "loss": 0.0039, + "num_tokens": 2306580.0, + "reward": 0.25, + "reward_std": 0.28867512941360474, + "rewards/compute_reward/mean": 0.25, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 210 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0844, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.7109271287918091, + "kl": 1.0758344233036041, + "learning_rate": 1.4383711467890773e-06, + "loss": 0.0043, + "num_tokens": 2315460.0, + "reward": 0.5, + "reward_std": 0.0, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 211 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0848, + "frac_reward_zero_std": 0.25, + "grad_norm": 38.93098449707031, + "kl": 1.124314084649086, + "learning_rate": 1.4320857488019824e-06, + "loss": 0.0045, + "num_tokens": 2324340.0, + "reward": 0.4375, + "reward_std": 0.41367512941360474, + "rewards/compute_reward/mean": 0.4375, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 212 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0852, + "frac_reward_zero_std": 0.0, + "grad_norm": 41.04912567138672, + "kl": 1.214218646287918, + "learning_rate": 1.4257792915650725e-06, + "loss": 0.0049, + "num_tokens": 2333220.0, + "reward": 0.375, + "reward_std": 0.5386751294136047, + "rewards/compute_reward/mean": 0.375, + "rewards/compute_reward/std": 0.5, + "step": 213 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0856, + "frac_reward_zero_std": 0.75, + "grad_norm": 27.365114212036133, + "kl": 1.4852878749370575, + "learning_rate": 1.419452082446177e-06, + "loss": 0.0059, + "num_tokens": 2342100.0, + "reward": 0.5625, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.5625, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 214 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.086, + "frac_reward_zero_std": 0.5, + "grad_norm": 41.17222595214844, + "kl": 1.54292231798172, + "learning_rate": 1.4131044298245418e-06, + "loss": 0.0062, + "num_tokens": 2350980.0, + "reward": 0.375, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.375, + "rewards/compute_reward/std": 0.5, + "step": 215 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0864, + "frac_reward_zero_std": 0.25, + "grad_norm": 40.92033386230469, + "kl": 1.4181734025478363, + "learning_rate": 1.4067366430758004e-06, + "loss": 0.0057, + "num_tokens": 2359860.0, + "reward": 0.5625, + "reward_std": 0.375, + "rewards/compute_reward/mean": 0.5625, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 216 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0868, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.1861588954925537, + "kl": 1.296187847852707, + "learning_rate": 1.400349032556895e-06, + "loss": 0.0052, + "num_tokens": 2368740.0, + "reward": 0.5, + "reward_std": 0.0, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 217 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0872, + "frac_reward_zero_std": 0.25, + "grad_norm": 37.92814636230469, + "kl": 1.2197140753269196, + "learning_rate": 1.393941909590951e-06, + "loss": 0.0049, + "num_tokens": 2377620.0, + "reward": 0.375, + "reward_std": 0.39433756470680237, + "rewards/compute_reward/mean": 0.375, + "rewards/compute_reward/std": 0.5, + "step": 218 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0876, + "frac_reward_zero_std": 0.25, + "grad_norm": 46.5506706237793, + "kl": 1.2918847501277924, + "learning_rate": 1.3875155864521028e-06, + "loss": 0.0052, + "num_tokens": 2386500.0, + "reward": 0.375, + "reward_std": 0.39433756470680237, + "rewards/compute_reward/mean": 0.375, + "rewards/compute_reward/std": 0.5, + "step": 219 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.088, + "frac_reward_zero_std": 0.75, + "grad_norm": 29.01485252380371, + "kl": 1.2826491594314575, + "learning_rate": 1.3810703763502743e-06, + "loss": 0.0051, + "num_tokens": 2395380.0, + "reward": 0.375, + "reward_std": 0.14433756470680237, + "rewards/compute_reward/mean": 0.375, + "rewards/compute_reward/std": 0.5, + "step": 220 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0884, + "frac_reward_zero_std": 0.5, + "grad_norm": 29.103654861450195, + "kl": 1.2425887286663055, + "learning_rate": 1.374606593415912e-06, + "loss": 0.005, + "num_tokens": 2404260.0, + "reward": 0.25, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.25, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 221 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0888, + "frac_reward_zero_std": 0.75, + "grad_norm": 21.997817993164062, + "kl": 1.2478073239326477, + "learning_rate": 1.3681245526846781e-06, + "loss": 0.005, + "num_tokens": 2413140.0, + "reward": 0.5625, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.5625, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 222 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0892, + "frac_reward_zero_std": 1.0, + "grad_norm": 1.3906086683273315, + "kl": 1.2335122525691986, + "learning_rate": 1.361624570082092e-06, + "loss": 0.0049, + "num_tokens": 2422020.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/compute_reward/mean": 0.0, + "rewards/compute_reward/std": 0.0, + "step": 223 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0896, + "frac_reward_zero_std": 0.5, + "grad_norm": 31.229379653930664, + "kl": 1.1545628011226654, + "learning_rate": 1.3551069624081371e-06, + "loss": 0.0046, + "num_tokens": 2430900.0, + "reward": 0.3125, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.3125, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 224 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.09, + "frac_reward_zero_std": 0.25, + "grad_norm": 37.78306198120117, + "kl": 1.1417304575443268, + "learning_rate": 1.3485720473218152e-06, + "loss": 0.0046, + "num_tokens": 2439780.0, + "reward": 0.25, + "reward_std": 0.39433756470680237, + "rewards/compute_reward/mean": 0.25, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 225 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0904, + "frac_reward_zero_std": 1.0, + "grad_norm": 0.7812005877494812, + "kl": 1.2861113250255585, + "learning_rate": 1.3420201433256689e-06, + "loss": 0.0051, + "num_tokens": 2448660.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/compute_reward/mean": 0.0, + "rewards/compute_reward/std": 0.0, + "step": 226 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0908, + "frac_reward_zero_std": 0.5, + "grad_norm": 37.55674743652344, + "kl": 1.3373107016086578, + "learning_rate": 1.3354515697502551e-06, + "loss": 0.0053, + "num_tokens": 2457540.0, + "reward": 0.5, + "reward_std": 0.28867512941360474, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 227 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0912, + "frac_reward_zero_std": 1.0, + "grad_norm": 1.6536084413528442, + "kl": 1.4782708883285522, + "learning_rate": 1.3288666467385831e-06, + "loss": 0.0059, + "num_tokens": 2466420.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/compute_reward/mean": 0.0, + "rewards/compute_reward/std": 0.0, + "step": 228 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0916, + "frac_reward_zero_std": 0.75, + "grad_norm": 21.825687408447266, + "kl": 1.3577501475811005, + "learning_rate": 1.3222656952305111e-06, + "loss": 0.0054, + "num_tokens": 2475300.0, + "reward": 0.3125, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.3125, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 229 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.092, + "frac_reward_zero_std": 0.75, + "grad_norm": 22.898921966552734, + "kl": 1.3672588467597961, + "learning_rate": 1.3156490369471024e-06, + "loss": 0.0055, + "num_tokens": 2484180.0, + "reward": 0.4375, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.4375, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 230 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0924, + "frac_reward_zero_std": 0.5, + "grad_norm": 36.318870544433594, + "kl": 1.4862727522850037, + "learning_rate": 1.3090169943749473e-06, + "loss": 0.0059, + "num_tokens": 2493060.0, + "reward": 0.625, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.625, + "rewards/compute_reward/std": 0.5, + "step": 231 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0928, + "frac_reward_zero_std": 0.75, + "grad_norm": 24.24749755859375, + "kl": 1.3374225199222565, + "learning_rate": 1.3023698907504446e-06, + "loss": 0.0053, + "num_tokens": 2501940.0, + "reward": 0.375, + "reward_std": 0.14433756470680237, + "rewards/compute_reward/mean": 0.375, + "rewards/compute_reward/std": 0.5, + "step": 232 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0932, + "frac_reward_zero_std": 0.25, + "grad_norm": 43.74175262451172, + "kl": 1.3122918009757996, + "learning_rate": 1.2957080500440467e-06, + "loss": 0.0052, + "num_tokens": 2510820.0, + "reward": 0.5625, + "reward_std": 0.41367512941360474, + "rewards/compute_reward/mean": 0.5625, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 233 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0936, + "frac_reward_zero_std": 0.75, + "grad_norm": 27.245943069458008, + "kl": 1.3207648694515228, + "learning_rate": 1.2890317969444716e-06, + "loss": 0.0053, + "num_tokens": 2519700.0, + "reward": 0.3125, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.3125, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 234 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.094, + "frac_reward_zero_std": 0.5, + "grad_norm": 39.74917984008789, + "kl": 1.4372384995222092, + "learning_rate": 1.2823414568428766e-06, + "loss": 0.0057, + "num_tokens": 2528580.0, + "reward": 0.3125, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.3125, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 235 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0944, + "frac_reward_zero_std": 0.5, + "grad_norm": 39.7496223449707, + "kl": 1.1310810148715973, + "learning_rate": 1.275637355816999e-06, + "loss": 0.0045, + "num_tokens": 2537460.0, + "reward": 0.4375, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.4375, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 236 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0948, + "frac_reward_zero_std": 0.25, + "grad_norm": 29.689462661743164, + "kl": 1.105551227927208, + "learning_rate": 1.2689198206152656e-06, + "loss": 0.0044, + "num_tokens": 2546340.0, + "reward": 0.75, + "reward_std": 0.39433756470680237, + "rewards/compute_reward/mean": 0.75, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 237 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0952, + "frac_reward_zero_std": 0.75, + "grad_norm": 19.529863357543945, + "kl": 1.0155319422483444, + "learning_rate": 1.2621891786408648e-06, + "loss": 0.0041, + "num_tokens": 2555220.0, + "reward": 0.375, + "reward_std": 0.14433756470680237, + "rewards/compute_reward/mean": 0.375, + "rewards/compute_reward/std": 0.5, + "step": 238 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0956, + "frac_reward_zero_std": 0.5, + "grad_norm": 40.87265396118164, + "kl": 1.1060357093811035, + "learning_rate": 1.2554457579357905e-06, + "loss": 0.0044, + "num_tokens": 2564100.0, + "reward": 0.5, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 239 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.096, + "frac_reward_zero_std": 0.5, + "grad_norm": 43.7393798828125, + "kl": 1.0452822744846344, + "learning_rate": 1.2486898871648551e-06, + "loss": 0.0042, + "num_tokens": 2572980.0, + "reward": 0.25, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.25, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 240 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0964, + "frac_reward_zero_std": 0.5, + "grad_norm": 42.85210418701172, + "kl": 1.2131805717945099, + "learning_rate": 1.2419218955996676e-06, + "loss": 0.0049, + "num_tokens": 2581860.0, + "reward": 0.25, + "reward_std": 0.28867512941360474, + "rewards/compute_reward/mean": 0.25, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 241 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0968, + "frac_reward_zero_std": 0.75, + "grad_norm": 25.9547119140625, + "kl": 1.0465219914913177, + "learning_rate": 1.23514211310259e-06, + "loss": 0.0042, + "num_tokens": 2590740.0, + "reward": 0.1875, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.1875, + "rewards/compute_reward/std": 0.40311288833618164, + "step": 242 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0972, + "frac_reward_zero_std": 0.5, + "grad_norm": 45.28058624267578, + "kl": 1.167938232421875, + "learning_rate": 1.2283508701106558e-06, + "loss": 0.0047, + "num_tokens": 2599620.0, + "reward": 0.125, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.125, + "rewards/compute_reward/std": 0.3415650427341461, + "step": 243 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0976, + "frac_reward_zero_std": 0.25, + "grad_norm": 60.2187614440918, + "kl": 0.9804764240980148, + "learning_rate": 1.2215484976194675e-06, + "loss": 0.0039, + "num_tokens": 2608500.0, + "reward": 0.375, + "reward_std": 0.39433756470680237, + "rewards/compute_reward/mean": 0.375, + "rewards/compute_reward/std": 0.5, + "step": 244 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.098, + "frac_reward_zero_std": 0.75, + "grad_norm": 29.929412841796875, + "kl": 1.051619529724121, + "learning_rate": 1.2147353271670632e-06, + "loss": 0.0042, + "num_tokens": 2617380.0, + "reward": 0.375, + "reward_std": 0.14433756470680237, + "rewards/compute_reward/mean": 0.375, + "rewards/compute_reward/std": 0.5, + "step": 245 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0984, + "frac_reward_zero_std": 0.5, + "grad_norm": 38.48051452636719, + "kl": 1.0911252945661545, + "learning_rate": 1.207911690817759e-06, + "loss": 0.0044, + "num_tokens": 2626260.0, + "reward": 0.5625, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.5625, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 246 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0988, + "frac_reward_zero_std": 0.75, + "grad_norm": 24.381433486938477, + "kl": 1.092143639922142, + "learning_rate": 1.2010779211459648e-06, + "loss": 0.0044, + "num_tokens": 2635140.0, + "reward": 0.4375, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.4375, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 247 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0992, + "frac_reward_zero_std": 0.5, + "grad_norm": 47.642539978027344, + "kl": 0.852707639336586, + "learning_rate": 1.194234351219972e-06, + "loss": 0.0034, + "num_tokens": 2644020.0, + "reward": 0.75, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.75, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 248 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.0996, + "frac_reward_zero_std": 0.5, + "grad_norm": 37.34217071533203, + "kl": 1.0041948407888412, + "learning_rate": 1.1873813145857248e-06, + "loss": 0.004, + "num_tokens": 2652900.0, + "reward": 0.4375, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.4375, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 249 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1, + "frac_reward_zero_std": 0.5, + "grad_norm": 27.347667694091797, + "kl": 0.9091732352972031, + "learning_rate": 1.18051914525056e-06, + "loss": 0.0036, + "num_tokens": 2661780.0, + "reward": 0.8125, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.8125, + "rewards/compute_reward/std": 0.40311288833618164, + "step": 250 + }, + { + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1, + "eval_loss": 0.0010513612069189548, + "eval_runtime": 5613.4422, + "eval_samples_per_second": 0.036, + "eval_steps_per_second": 0.009, + "frac_reward_zero_std": 0.7, + "num_tokens": 2772780.0, + "reward": 0.39, + "reward_std": 0.15773502588272095, + "rewards/compute_reward/mean": 0.39, + "rewards/compute_reward/std": 0.15773502588272095, + "step": 250 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1004, + "frac_reward_zero_std": 0.5, + "grad_norm": 22.235748291015625, + "kl": 1.82113316655159, + "learning_rate": 1.1736481776669305e-06, + "loss": 0.0073, + "num_tokens": 2781660.0, + "reward": 0.5, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 251 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1008, + "frac_reward_zero_std": 0.75, + "grad_norm": 39.217552185058594, + "kl": 1.126240998506546, + "learning_rate": 1.1667687467161023e-06, + "loss": -0.212, + "num_tokens": 2790540.0, + "reward": 0.375, + "reward_std": 0.14433756470680237, + "rewards/compute_reward/mean": 0.375, + "rewards/compute_reward/std": 0.5, + "step": 252 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1012, + "frac_reward_zero_std": 0.5, + "grad_norm": 34.31150436401367, + "kl": 1.3839496076107025, + "learning_rate": 1.1598811876918349e-06, + "loss": 0.0055, + "num_tokens": 2799420.0, + "reward": 0.25, + "reward_std": 0.28867512941360474, + "rewards/compute_reward/mean": 0.25, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 253 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1016, + "frac_reward_zero_std": 0.25, + "grad_norm": 39.29949188232422, + "kl": 0.9901166260242462, + "learning_rate": 1.1529858362840382e-06, + "loss": 0.5619, + "num_tokens": 2808300.0, + "reward": 0.5, + "reward_std": 0.39433756470680237, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 254 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.102, + "frac_reward_zero_std": 1.0, + "grad_norm": 16.917476654052734, + "kl": 0.8453292548656464, + "learning_rate": 1.1460830285624116e-06, + "loss": -0.3381, + "num_tokens": 2817180.0, + "reward": 0.75, + "reward_std": 0.0, + "rewards/compute_reward/mean": 0.75, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 255 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1024, + "frac_reward_zero_std": 0.5, + "grad_norm": 49.95372772216797, + "kl": 1.1023323237895966, + "learning_rate": 1.1391731009600653e-06, + "loss": -0.1206, + "num_tokens": 2826060.0, + "reward": 0.5, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 256 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1028, + "frac_reward_zero_std": 0.75, + "grad_norm": 31.83892822265625, + "kl": 1.1208092421293259, + "learning_rate": 1.1322563902571225e-06, + "loss": 0.1295, + "num_tokens": 2834940.0, + "reward": 0.1875, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.1875, + "rewards/compute_reward/std": 0.40311288833618164, + "step": 257 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1032, + "frac_reward_zero_std": 0.5, + "grad_norm": 19.977169036865234, + "kl": 0.9258421361446381, + "learning_rate": 1.1253332335643042e-06, + "loss": -0.3712, + "num_tokens": 2843820.0, + "reward": 0.3125, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.3125, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 258 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1036, + "frac_reward_zero_std": 0.25, + "grad_norm": 45.17646026611328, + "kl": 1.021436646580696, + "learning_rate": 1.1184039683065012e-06, + "loss": 0.504, + "num_tokens": 2852700.0, + "reward": 0.3125, + "reward_std": 0.375, + "rewards/compute_reward/mean": 0.3125, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 259 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.104, + "frac_reward_zero_std": 0.75, + "grad_norm": 29.35318374633789, + "kl": 0.7680477648973465, + "learning_rate": 1.1114689322063254e-06, + "loss": -0.1219, + "num_tokens": 2861580.0, + "reward": 0.4375, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.4375, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 260 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1044, + "frac_reward_zero_std": 0.25, + "grad_norm": 28.826967239379883, + "kl": 0.9056837409734726, + "learning_rate": 1.1045284632676535e-06, + "loss": 0.2201, + "num_tokens": 2870460.0, + "reward": 0.3125, + "reward_std": 0.41367512941360474, + "rewards/compute_reward/mean": 0.3125, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 261 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1048, + "frac_reward_zero_std": 0.75, + "grad_norm": 30.201587677001953, + "kl": 1.171682357788086, + "learning_rate": 1.0975828997591495e-06, + "loss": -0.2118, + "num_tokens": 2879340.0, + "reward": 0.375, + "reward_std": 0.14433756470680237, + "rewards/compute_reward/mean": 0.375, + "rewards/compute_reward/std": 0.5, + "step": 262 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1052, + "frac_reward_zero_std": 0.25, + "grad_norm": 31.657508850097656, + "kl": 0.9881609976291656, + "learning_rate": 1.0906325801977803e-06, + "loss": 0.1289, + "num_tokens": 2888220.0, + "reward": 0.4375, + "reward_std": 0.375, + "rewards/compute_reward/mean": 0.4375, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 263 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1056, + "frac_reward_zero_std": 1.0, + "grad_norm": 13.9951810836792, + "kl": 1.0576434880495071, + "learning_rate": 1.0836778433323157e-06, + "loss": -0.1207, + "num_tokens": 2897100.0, + "reward": 0.5, + "reward_std": 0.0, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 264 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.106, + "frac_reward_zero_std": 0.25, + "grad_norm": 24.13888931274414, + "kl": 0.7200263738632202, + "learning_rate": 1.0767190281268186e-06, + "loss": 0.2193, + "num_tokens": 2905980.0, + "reward": 0.625, + "reward_std": 0.39433756470680237, + "rewards/compute_reward/mean": 0.625, + "rewards/compute_reward/std": 0.5, + "step": 265 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1064, + "frac_reward_zero_std": 0.25, + "grad_norm": 47.248741149902344, + "kl": 0.9038263559341431, + "learning_rate": 1.069756473744125e-06, + "loss": -0.2463, + "num_tokens": 2914860.0, + "reward": 0.375, + "reward_std": 0.39433756470680237, + "rewards/compute_reward/mean": 0.375, + "rewards/compute_reward/std": 0.5, + "step": 266 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1068, + "frac_reward_zero_std": 0.5, + "grad_norm": 30.434354782104492, + "kl": 0.8619530349969864, + "learning_rate": 1.0627905195293135e-06, + "loss": 0.2869, + "num_tokens": 2923740.0, + "reward": 0.625, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.625, + "rewards/compute_reward/std": 0.5, + "step": 267 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1072, + "frac_reward_zero_std": 0.75, + "grad_norm": 14.322568893432617, + "kl": 0.8955692648887634, + "learning_rate": 1.055821504993164e-06, + "loss": -0.3713, + "num_tokens": 2932620.0, + "reward": 0.0625, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.0625, + "rewards/compute_reward/std": 0.25, + "step": 268 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1076, + "frac_reward_zero_std": 0.25, + "grad_norm": 38.34691619873047, + "kl": 0.8839395940303802, + "learning_rate": 1.0488497697956134e-06, + "loss": 0.0035, + "num_tokens": 2941500.0, + "reward": 0.5, + "reward_std": 0.39433756470680237, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 269 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.108, + "frac_reward_zero_std": 0.5, + "grad_norm": 34.70941925048828, + "kl": 0.8146094232797623, + "learning_rate": 1.0418756537291995e-06, + "loss": 0.4697, + "num_tokens": 2950380.0, + "reward": 0.5625, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.5625, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 270 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1084, + "frac_reward_zero_std": 0.5, + "grad_norm": 22.19832992553711, + "kl": 0.7530200630426407, + "learning_rate": 1.034899496702501e-06, + "loss": -0.2135, + "num_tokens": 2959260.0, + "reward": 0.625, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.625, + "rewards/compute_reward/std": 0.5, + "step": 271 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1088, + "frac_reward_zero_std": 0.5, + "grad_norm": 34.179874420166016, + "kl": 0.8426708728075027, + "learning_rate": 1.0279216387235689e-06, + "loss": -0.1216, + "num_tokens": 2968140.0, + "reward": 0.1875, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.1875, + "rewards/compute_reward/std": 0.40311288833618164, + "step": 272 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1092, + "frac_reward_zero_std": 0.25, + "grad_norm": 20.24189567565918, + "kl": 0.8162007182836533, + "learning_rate": 1.020942419883357e-06, + "loss": -0.0882, + "num_tokens": 2977020.0, + "reward": 0.4375, + "reward_std": 0.41367512941360474, + "rewards/compute_reward/mean": 0.4375, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 273 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1096, + "frac_reward_zero_std": 0.25, + "grad_norm": 28.002971649169922, + "kl": 0.669597715139389, + "learning_rate": 1.0139621803391454e-06, + "loss": 0.0942, + "num_tokens": 2985900.0, + "reward": 0.3125, + "reward_std": 0.41367512941360474, + "rewards/compute_reward/mean": 0.3125, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 274 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.11, + "frac_reward_zero_std": 0.75, + "grad_norm": 19.8535213470459, + "kl": 0.8049573004245758, + "learning_rate": 1.0069812602979615e-06, + "loss": 0.2197, + "num_tokens": 2994780.0, + "reward": 0.625, + "reward_std": 0.14433756470680237, + "rewards/compute_reward/mean": 0.625, + "rewards/compute_reward/std": 0.5, + "step": 275 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1104, + "frac_reward_zero_std": 0.75, + "grad_norm": 17.800783157348633, + "kl": 0.8933698236942291, + "learning_rate": 1e-06, + "loss": -0.2129, + "num_tokens": 3003660.0, + "reward": 0.6875, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.6875, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 276 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1108, + "frac_reward_zero_std": 0.75, + "grad_norm": 12.544022560119629, + "kl": 0.7986416071653366, + "learning_rate": 9.930187397020384e-07, + "loss": 0.1282, + "num_tokens": 3012540.0, + "reward": 0.1875, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.1875, + "rewards/compute_reward/std": 0.40311288833618164, + "step": 277 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1112, + "frac_reward_zero_std": 1.0, + "grad_norm": 3.61922550201416, + "kl": 0.7199362516403198, + "learning_rate": 9.860378196608547e-07, + "loss": -0.1221, + "num_tokens": 3021420.0, + "reward": 0.75, + "reward_std": 0.0, + "rewards/compute_reward/mean": 0.75, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 278 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1116, + "frac_reward_zero_std": 0.5, + "grad_norm": 19.929853439331055, + "kl": 0.7778548300266266, + "learning_rate": 9.790575801166431e-07, + "loss": 0.2531, + "num_tokens": 3030300.0, + "reward": 0.375, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.375, + "rewards/compute_reward/std": 0.5, + "step": 279 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.112, + "frac_reward_zero_std": 0.5, + "grad_norm": 17.565427780151367, + "kl": 0.8007985502481461, + "learning_rate": 9.720783612764313e-07, + "loss": -0.3717, + "num_tokens": 3039180.0, + "reward": 0.6875, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.6875, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 280 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1124, + "frac_reward_zero_std": 0.75, + "grad_norm": 8.918659210205078, + "kl": 0.67110276222229, + "learning_rate": 9.651005032974993e-07, + "loss": 0.1277, + "num_tokens": 3048060.0, + "reward": 0.1875, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.1875, + "rewards/compute_reward/std": 0.40311288833618164, + "step": 281 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1128, + "frac_reward_zero_std": 0.75, + "grad_norm": 7.046802997589111, + "kl": 0.6922563761472702, + "learning_rate": 9.581243462708005e-07, + "loss": 0.1277, + "num_tokens": 3056940.0, + "reward": 0.6875, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.6875, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 282 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1132, + "frac_reward_zero_std": 0.5, + "grad_norm": 16.287311553955078, + "kl": 0.7062659412622452, + "learning_rate": 9.511502302043867e-07, + "loss": -0.2471, + "num_tokens": 3065820.0, + "reward": 0.375, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.375, + "rewards/compute_reward/std": 0.5, + "step": 283 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1136, + "frac_reward_zero_std": 0.5, + "grad_norm": 14.158629417419434, + "kl": 0.8502112925052643, + "learning_rate": 9.441784950068361e-07, + "loss": 0.7198, + "num_tokens": 3074700.0, + "reward": 0.6875, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.6875, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 284 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.114, + "frac_reward_zero_std": 1.0, + "grad_norm": 11.075551986694336, + "kl": 0.6977677792310715, + "learning_rate": 9.372094804706866e-07, + "loss": -0.5886, + "num_tokens": 3083580.0, + "reward": 0.25, + "reward_std": 0.0, + "rewards/compute_reward/mean": 0.25, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 285 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1144, + "frac_reward_zero_std": 0.75, + "grad_norm": 11.198731422424316, + "kl": 0.6550881266593933, + "learning_rate": 9.302435262558747e-07, + "loss": -0.1224, + "num_tokens": 3092460.0, + "reward": 0.0625, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.0625, + "rewards/compute_reward/std": 0.25, + "step": 286 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1148, + "frac_reward_zero_std": 0.75, + "grad_norm": 14.440635681152344, + "kl": 0.6763903424143791, + "learning_rate": 9.232809718731813e-07, + "loss": 0.1277, + "num_tokens": 3101340.0, + "reward": 0.6875, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.6875, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 287 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1152, + "frac_reward_zero_std": 0.5, + "grad_norm": 14.166817665100098, + "kl": 0.8325612992048264, + "learning_rate": 9.163221566676847e-07, + "loss": 0.2198, + "num_tokens": 3110220.0, + "reward": 0.75, + "reward_std": 0.28867512941360474, + "rewards/compute_reward/mean": 0.75, + "rewards/compute_reward/std": 0.44721361994743347, + "step": 288 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1156, + "frac_reward_zero_std": 0.25, + "grad_norm": 20.33856773376465, + "kl": 0.6119056344032288, + "learning_rate": 9.093674198022199e-07, + "loss": -0.397, + "num_tokens": 3119100.0, + "reward": 0.5, + "reward_std": 0.39433756470680237, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 289 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.116, + "frac_reward_zero_std": 0.5, + "grad_norm": 17.874990463256836, + "kl": 0.8161603510379791, + "learning_rate": 9.024171002408506e-07, + "loss": -0.0302, + "num_tokens": 3127980.0, + "reward": 0.1875, + "reward_std": 0.26933756470680237, + "rewards/compute_reward/mean": 0.1875, + "rewards/compute_reward/std": 0.40311288833618164, + "step": 290 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1164, + "frac_reward_zero_std": 0.75, + "grad_norm": 9.15800952911377, + "kl": 0.6517329961061478, + "learning_rate": 8.954715367323466e-07, + "loss": 0.594, + "num_tokens": 3136860.0, + "reward": 0.5625, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.5625, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 291 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1168, + "frac_reward_zero_std": 1.0, + "grad_norm": 15.840858459472656, + "kl": 0.6931683421134949, + "learning_rate": 8.885310677936746e-07, + "loss": -0.3722, + "num_tokens": 3145740.0, + "reward": 0.5, + "reward_std": 0.0, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 292 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1172, + "frac_reward_zero_std": 0.25, + "grad_norm": 15.857455253601074, + "kl": 0.5569572299718857, + "learning_rate": 8.81596031693499e-07, + "loss": 0.2187, + "num_tokens": 3154620.0, + "reward": 0.4375, + "reward_std": 0.41367512941360474, + "rewards/compute_reward/mean": 0.4375, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 293 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1176, + "frac_reward_zero_std": 0.5, + "grad_norm": 14.625517845153809, + "kl": 0.7273398488759995, + "learning_rate": 8.746667664356955e-07, + "loss": -0.6465, + "num_tokens": 3163500.0, + "reward": 0.5, + "reward_std": 0.28867512941360474, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 294 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.118, + "frac_reward_zero_std": 0.25, + "grad_norm": 225.3708038330078, + "kl": 0.7756715416908264, + "learning_rate": 8.677436097428774e-07, + "loss": 0.0031, + "num_tokens": 3172380.0, + "reward": 0.6875, + "reward_std": 0.41367512941360474, + "rewards/compute_reward/mean": 0.6875, + "rewards/compute_reward/std": 0.4787135720252991, + "step": 295 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1184, + "frac_reward_zero_std": 0.0, + "grad_norm": 24.645357131958008, + "kl": 0.7122261449694633, + "learning_rate": 8.608268990399348e-07, + "loss": 0.1858, + "num_tokens": 3181260.0, + "reward": 0.5, + "reward_std": 0.5, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 296 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1188, + "frac_reward_zero_std": 0.75, + "grad_norm": 12.578080177307129, + "kl": 0.8148588389158249, + "learning_rate": 8.539169714375885e-07, + "loss": 0.4697, + "num_tokens": 3190140.0, + "reward": 0.375, + "reward_std": 0.14433756470680237, + "rewards/compute_reward/mean": 0.375, + "rewards/compute_reward/std": 0.5, + "step": 297 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1192, + "frac_reward_zero_std": 0.5, + "grad_norm": 11.741436004638672, + "kl": 0.7392874956130981, + "learning_rate": 8.47014163715962e-07, + "loss": -0.3385, + "num_tokens": 3199020.0, + "reward": 0.5, + "reward_std": 0.25, + "rewards/compute_reward/mean": 0.5, + "rewards/compute_reward/std": 0.5163977742195129, + "step": 298 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.1196, + "frac_reward_zero_std": 0.25, + "grad_norm": 21.388893127441406, + "kl": 0.8566679060459137, + "learning_rate": 8.401188123081652e-07, + "loss": 0.2199, + "num_tokens": 3207900.0, + "reward": 0.4375, + "reward_std": 0.41367512941360474, + "rewards/compute_reward/mean": 0.4375, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 299 + }, + { + "completion_length": 512.0, + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.12, + "frac_reward_zero_std": 0.75, + "grad_norm": 13.048949241638184, + "kl": 0.8310280442237854, + "learning_rate": 8.332312532838978e-07, + "loss": -0.3381, + "num_tokens": 3216780.0, + "reward": 0.5625, + "reward_std": 0.125, + "rewards/compute_reward/mean": 0.5625, + "rewards/compute_reward/std": 0.5123475790023804, + "step": 300 + }, + { + "completions/clipped_ratio": 1.0, + "completions/max_length": 512.0, + "completions/max_terminated_length": 0.0, + "completions/mean_length": 512.0, + "completions/mean_terminated_length": 0.0, + "completions/min_length": 512.0, + "completions/min_terminated_length": 0.0, + "epoch": 0.12, + "eval_loss": 0.002101506805047393, + "eval_runtime": 7435.4156, + "eval_samples_per_second": 0.027, + "eval_steps_per_second": 0.007, + "frac_reward_zero_std": 0.68, + "num_tokens": 3327780.0, + "reward": 0.42, + "reward_std": 0.16618802070617675, + "rewards/compute_reward/mean": 0.42, + "rewards/compute_reward/std": 0.16618802070617675, + "step": 300 + } + ], + "logging_steps": 1, + "max_steps": 500, + "num_input_tokens_seen": 3327780, + "num_train_epochs": 1, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +}