File size: 11,988 Bytes
5c75558 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 |
{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 2.0,
"eval_steps": 100,
"global_step": 192,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.010416666666666666,
"grad_norm": 284.73684625892474,
"learning_rate": 4.386080498729062e-08,
"logits/chosen": -2.590585231781006,
"logits/rejected": -2.5664222240448,
"logps/chosen": -80.29847717285156,
"logps/rejected": -53.10200881958008,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1
},
{
"epoch": 0.10416666666666667,
"grad_norm": 269.4644096553184,
"learning_rate": 4.386080498729062e-07,
"logits/chosen": -2.5559744834899902,
"logits/rejected": -2.5380678176879883,
"logps/chosen": -87.90437316894531,
"logps/rejected": -80.99159240722656,
"loss": 0.698,
"rewards/accuracies": 0.2013888955116272,
"rewards/chosen": 0.004661789629608393,
"rewards/margins": -0.01561733242124319,
"rewards/rejected": 0.02027912251651287,
"step": 10
},
{
"epoch": 0.20833333333333334,
"grad_norm": 171.18434231702736,
"learning_rate": 8.772160997458124e-07,
"logits/chosen": -2.604264736175537,
"logits/rejected": -2.5549380779266357,
"logps/chosen": -102.35679626464844,
"logps/rejected": -89.08101654052734,
"loss": 0.6597,
"rewards/accuracies": 0.35624998807907104,
"rewards/chosen": 0.4296715259552002,
"rewards/margins": 0.08247259259223938,
"rewards/rejected": 0.34719887375831604,
"step": 20
},
{
"epoch": 0.3125,
"grad_norm": 307.77121119192236,
"learning_rate": 8.320291172024512e-07,
"logits/chosen": -2.482062578201294,
"logits/rejected": -2.4983456134796143,
"logps/chosen": -64.65339660644531,
"logps/rejected": -73.08040618896484,
"loss": 0.7129,
"rewards/accuracies": 0.3499999940395355,
"rewards/chosen": 1.2031724452972412,
"rewards/margins": 0.40556639432907104,
"rewards/rejected": 0.7976058721542358,
"step": 30
},
{
"epoch": 0.4166666666666667,
"grad_norm": 247.69654790448098,
"learning_rate": 7.868421346590901e-07,
"logits/chosen": -2.5462698936462402,
"logits/rejected": -2.5348854064941406,
"logps/chosen": -68.5112075805664,
"logps/rejected": -67.58045196533203,
"loss": 0.7089,
"rewards/accuracies": 0.26875001192092896,
"rewards/chosen": 1.5711699724197388,
"rewards/margins": 0.21717211604118347,
"rewards/rejected": 1.353997826576233,
"step": 40
},
{
"epoch": 0.5208333333333334,
"grad_norm": 245.28947413610663,
"learning_rate": 7.416551521157288e-07,
"logits/chosen": -2.4909956455230713,
"logits/rejected": -2.5039918422698975,
"logps/chosen": -47.20299530029297,
"logps/rejected": -55.72983932495117,
"loss": 0.7414,
"rewards/accuracies": 0.24375000596046448,
"rewards/chosen": 1.5983082056045532,
"rewards/margins": 0.22424061596393585,
"rewards/rejected": 1.3740675449371338,
"step": 50
},
{
"epoch": 0.625,
"grad_norm": 185.80480485868813,
"learning_rate": 6.964681695723677e-07,
"logits/chosen": -2.6212210655212402,
"logits/rejected": -2.6057589054107666,
"logps/chosen": -73.73361206054688,
"logps/rejected": -75.0650863647461,
"loss": 0.7278,
"rewards/accuracies": 0.3125,
"rewards/chosen": 2.306000232696533,
"rewards/margins": 0.4594852924346924,
"rewards/rejected": 1.8465149402618408,
"step": 60
},
{
"epoch": 0.7291666666666666,
"grad_norm": 238.44222468726358,
"learning_rate": 6.512811870290064e-07,
"logits/chosen": -2.616278886795044,
"logits/rejected": -2.6074726581573486,
"logps/chosen": -91.4566421508789,
"logps/rejected": -81.8633041381836,
"loss": 0.7043,
"rewards/accuracies": 0.35624998807907104,
"rewards/chosen": 2.112656354904175,
"rewards/margins": 0.5624496936798096,
"rewards/rejected": 1.5502065420150757,
"step": 70
},
{
"epoch": 0.8333333333333334,
"grad_norm": 359.10559072083385,
"learning_rate": 6.060942044856453e-07,
"logits/chosen": -2.593449115753174,
"logits/rejected": -2.54801607131958,
"logps/chosen": -80.93285369873047,
"logps/rejected": -76.7210693359375,
"loss": 0.7058,
"rewards/accuracies": 0.41874998807907104,
"rewards/chosen": 2.0366339683532715,
"rewards/margins": 0.886386513710022,
"rewards/rejected": 1.1502474546432495,
"step": 80
},
{
"epoch": 0.9375,
"grad_norm": 283.4561861873084,
"learning_rate": 5.609072219422841e-07,
"logits/chosen": -2.5106282234191895,
"logits/rejected": -2.506967306137085,
"logps/chosen": -51.3546028137207,
"logps/rejected": -61.58504104614258,
"loss": 0.7583,
"rewards/accuracies": 0.23125000298023224,
"rewards/chosen": 1.134612798690796,
"rewards/margins": 0.32073575258255005,
"rewards/rejected": 0.8138772249221802,
"step": 90
},
{
"epoch": 1.0416666666666667,
"grad_norm": 21.926819298392022,
"learning_rate": 5.157202393989229e-07,
"logits/chosen": -2.4917004108428955,
"logits/rejected": -2.476295232772827,
"logps/chosen": -65.5259780883789,
"logps/rejected": -74.4820327758789,
"loss": 0.5775,
"rewards/accuracies": 0.40625,
"rewards/chosen": 3.455796718597412,
"rewards/margins": 4.517679691314697,
"rewards/rejected": -1.0618823766708374,
"step": 100
},
{
"epoch": 1.0416666666666667,
"eval_logits/chosen": -2.5355653762817383,
"eval_logits/rejected": -2.5199732780456543,
"eval_logps/chosen": -72.7430191040039,
"eval_logps/rejected": -80.81999206542969,
"eval_loss": 0.7711760401725769,
"eval_rewards/accuracies": 0.3313491940498352,
"eval_rewards/chosen": 0.7555689811706543,
"eval_rewards/margins": 0.6222997903823853,
"eval_rewards/rejected": 0.1332692801952362,
"eval_runtime": 113.7538,
"eval_samples_per_second": 17.582,
"eval_steps_per_second": 0.554,
"step": 100
},
{
"epoch": 1.1458333333333333,
"grad_norm": 11.001098983606164,
"learning_rate": 4.7053325685556164e-07,
"logits/chosen": -2.5152523517608643,
"logits/rejected": -2.542959690093994,
"logps/chosen": -55.308494567871094,
"logps/rejected": -93.90345001220703,
"loss": 0.3605,
"rewards/accuracies": 0.48124998807907104,
"rewards/chosen": 4.347947120666504,
"rewards/margins": 9.715516090393066,
"rewards/rejected": -5.367568492889404,
"step": 110
},
{
"epoch": 1.25,
"grad_norm": 5.506771482244887,
"learning_rate": 4.2534627431220053e-07,
"logits/chosen": -2.5518078804016113,
"logits/rejected": -2.5186076164245605,
"logps/chosen": -95.32207489013672,
"logps/rejected": -115.25175476074219,
"loss": 0.3819,
"rewards/accuracies": 0.581250011920929,
"rewards/chosen": 5.481812477111816,
"rewards/margins": 14.953575134277344,
"rewards/rejected": -9.471762657165527,
"step": 120
},
{
"epoch": 1.3541666666666667,
"grad_norm": 80.99035607988066,
"learning_rate": 3.801592917688393e-07,
"logits/chosen": -2.4438960552215576,
"logits/rejected": -2.44427490234375,
"logps/chosen": -80.55229187011719,
"logps/rejected": -110.3847885131836,
"loss": 0.3763,
"rewards/accuracies": 0.46875,
"rewards/chosen": 2.599090814590454,
"rewards/margins": 11.696044921875,
"rewards/rejected": -9.096954345703125,
"step": 130
},
{
"epoch": 1.4583333333333333,
"grad_norm": 81.79049224504786,
"learning_rate": 3.3497230922547816e-07,
"logits/chosen": -2.442150831222534,
"logits/rejected": -2.4587113857269287,
"logps/chosen": -47.28990936279297,
"logps/rejected": -80.22529602050781,
"loss": 0.3766,
"rewards/accuracies": 0.42500001192092896,
"rewards/chosen": 0.8634545207023621,
"rewards/margins": 7.3911895751953125,
"rewards/rejected": -6.527735233306885,
"step": 140
},
{
"epoch": 1.5625,
"grad_norm": 2.5199873819499934,
"learning_rate": 2.89785326682117e-07,
"logits/chosen": -2.4821057319641113,
"logits/rejected": -2.4555060863494873,
"logps/chosen": -67.4425277709961,
"logps/rejected": -90.55575561523438,
"loss": 0.3829,
"rewards/accuracies": 0.46875,
"rewards/chosen": 2.033623456954956,
"rewards/margins": 9.372200012207031,
"rewards/rejected": -7.338575839996338,
"step": 150
},
{
"epoch": 1.6666666666666665,
"grad_norm": 1.8477188844357901,
"learning_rate": 2.445983441387558e-07,
"logits/chosen": -2.4257683753967285,
"logits/rejected": -2.4348573684692383,
"logps/chosen": -72.42179870605469,
"logps/rejected": -111.45719146728516,
"loss": 0.3664,
"rewards/accuracies": 0.5375000238418579,
"rewards/chosen": 3.675116777420044,
"rewards/margins": 13.840252876281738,
"rewards/rejected": -10.165136337280273,
"step": 160
},
{
"epoch": 1.7708333333333335,
"grad_norm": 27.363913250696022,
"learning_rate": 1.9941136159539462e-07,
"logits/chosen": -2.3719429969787598,
"logits/rejected": -2.3617918491363525,
"logps/chosen": -68.02982330322266,
"logps/rejected": -105.62237548828125,
"loss": 0.394,
"rewards/accuracies": 0.53125,
"rewards/chosen": 3.107600212097168,
"rewards/margins": 12.890829086303711,
"rewards/rejected": -9.783227920532227,
"step": 170
},
{
"epoch": 1.875,
"grad_norm": 43.422774421532836,
"learning_rate": 1.5422437905203344e-07,
"logits/chosen": -2.4057819843292236,
"logits/rejected": -2.389813184738159,
"logps/chosen": -64.71813201904297,
"logps/rejected": -92.5549545288086,
"loss": 0.3757,
"rewards/accuracies": 0.48750001192092896,
"rewards/chosen": 3.084803581237793,
"rewards/margins": 11.271722793579102,
"rewards/rejected": -8.186918258666992,
"step": 180
},
{
"epoch": 1.9791666666666665,
"grad_norm": 2.3019013228739755,
"learning_rate": 1.0903739650867225e-07,
"logits/chosen": -2.454073429107666,
"logits/rejected": -2.4500691890716553,
"logps/chosen": -76.17430877685547,
"logps/rejected": -113.40323638916016,
"loss": 0.3748,
"rewards/accuracies": 0.5249999761581421,
"rewards/chosen": 5.310562610626221,
"rewards/margins": 15.156728744506836,
"rewards/rejected": -9.846165657043457,
"step": 190
},
{
"epoch": 2.0,
"step": 192,
"total_flos": 0.0,
"train_loss": 0.5438582648833593,
"train_runtime": 2038.8345,
"train_samples_per_second": 5.997,
"train_steps_per_second": 0.094
}
],
"logging_steps": 10,
"max_steps": 192,
"num_input_tokens_seen": 0,
"num_train_epochs": 2,
"save_steps": 100,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": true
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 8,
"trial_name": null,
"trial_params": null
}
|