Training in progress, step 88000, checkpoint

checkpoint-88000/trainer_state.json

@@ -11,9 +11,9 @@
     {
       "epoch": 2.5,
       "eval_loss": 7.232339382171631,
-      "eval_runtime": 1.6226,
-      "eval_samples_per_second": 1272.646,
-      "eval_steps_per_second": 80.118,
+      "eval_runtime": 3.1848,
+      "eval_samples_per_second": 648.395,
+      "eval_steps_per_second": 40.819,
       "step": 2000
     },
     {
@@ -26,17 +26,17 @@
     {
       "epoch": 5.0,
       "eval_loss": 5.976784706115723,
-      "eval_runtime": 1.6382,
-      "eval_samples_per_second": 1260.5,
-      "eval_steps_per_second": 79.353,
+      "eval_runtime": 3.0375,
+      "eval_samples_per_second": 679.836,
+      "eval_steps_per_second": 42.798,
       "step": 4000
     },
     {
       "epoch": 7.5,
       "eval_loss": 5.826813220977783,
-      "eval_runtime": 3.0373,
-      "eval_samples_per_second": 679.889,
-      "eval_steps_per_second": 42.802,
+      "eval_runtime": 2.9574,
+      "eval_samples_per_second": 698.238,
+      "eval_steps_per_second": 43.957,
       "step": 6000
     },
     {
@@ -49,17 +49,17 @@
     {
       "epoch": 10.0,
       "eval_loss": 5.744414806365967,
-      "eval_runtime": 3.0316,
-      "eval_samples_per_second": 681.153,
-      "eval_steps_per_second": 42.881,
+      "eval_runtime": 2.5747,
+      "eval_samples_per_second": 802.048,
+      "eval_steps_per_second": 50.492,
       "step": 8000
     },
     {
       "epoch": 12.5,
       "eval_loss": 5.670751571655273,
-      "eval_runtime": 3.1272,
-      "eval_samples_per_second": 660.333,
-      "eval_steps_per_second": 41.571,
+      "eval_runtime": 1.618,
+      "eval_samples_per_second": 1276.247,
+      "eval_steps_per_second": 80.345,
       "step": 10000
     },
     {
@@ -72,17 +72,17 @@
     {
       "epoch": 15.0,
       "eval_loss": 5.57132625579834,
-      "eval_runtime": 3.0078,
-      "eval_samples_per_second": 686.558,
-      "eval_steps_per_second": 43.222,
+      "eval_runtime": 2.872,
+      "eval_samples_per_second": 719.021,
+      "eval_steps_per_second": 45.265,
       "step": 12000
     },
     {
       "epoch": 17.5,
       "eval_loss": 5.526998043060303,
-      "eval_runtime": 3.0343,
-      "eval_samples_per_second": 680.548,
-      "eval_steps_per_second": 42.843,
+      "eval_runtime": 3.0079,
+      "eval_samples_per_second": 686.536,
+      "eval_steps_per_second": 43.22,
       "step": 14000
     },
     {
@@ -95,17 +95,17 @@
     {
       "epoch": 20.0,
       "eval_loss": 5.448328971862793,
-      "eval_runtime": 3.0113,
-      "eval_samples_per_second": 685.757,
-      "eval_steps_per_second": 43.171,
+      "eval_runtime": 2.9966,
+      "eval_samples_per_second": 689.104,
+      "eval_steps_per_second": 43.382,
       "step": 16000
     },
     {
       "epoch": 22.5,
       "eval_loss": 5.3926005363464355,
-      "eval_runtime": 3.0569,
-      "eval_samples_per_second": 675.527,
-      "eval_steps_per_second": 42.527,
+      "eval_runtime": 3.0264,
+      "eval_samples_per_second": 682.332,
+      "eval_steps_per_second": 42.955,
       "step": 18000
     },
     {
@@ -118,17 +118,17 @@
     {
       "epoch": 25.0,
       "eval_loss": 5.231536388397217,
-      "eval_runtime": 2.8975,
-      "eval_samples_per_second": 712.671,
-      "eval_steps_per_second": 44.865,
+      "eval_runtime": 3.0403,
+      "eval_samples_per_second": 679.22,
+      "eval_steps_per_second": 42.76,
       "step": 20000
     },
     {
       "epoch": 27.5,
       "eval_loss": 4.905914306640625,
-      "eval_runtime": 3.0834,
-      "eval_samples_per_second": 669.706,
-      "eval_steps_per_second": 42.161,
+      "eval_runtime": 2.9951,
+      "eval_samples_per_second": 689.451,
+      "eval_steps_per_second": 43.404,
       "step": 22000
     },
     {
@@ -141,17 +141,17 @@
     {
       "epoch": 30.0,
       "eval_loss": 4.167966842651367,
-      "eval_runtime": 2.6664,
-      "eval_samples_per_second": 774.461,
-      "eval_steps_per_second": 48.755,
+      "eval_runtime": 2.4346,
+      "eval_samples_per_second": 848.205,
+      "eval_steps_per_second": 53.398,
       "step": 24000
     },
     {
       "epoch": 32.5,
       "eval_loss": 3.6409149169921875,
-      "eval_runtime": 2.5685,
-      "eval_samples_per_second": 803.957,
-      "eval_steps_per_second": 50.612,
+      "eval_runtime": 2.8198,
+      "eval_samples_per_second": 732.313,
+      "eval_steps_per_second": 46.102,
       "step": 26000
     },
     {
@@ -164,17 +164,17 @@
     {
       "epoch": 35.0,
       "eval_loss": 3.2063941955566406,
-      "eval_runtime": 2.8723,
-      "eval_samples_per_second": 718.927,
-      "eval_steps_per_second": 45.259,
+      "eval_runtime": 2.8452,
+      "eval_samples_per_second": 725.772,
+      "eval_steps_per_second": 45.69,
       "step": 28000
     },
     {
       "epoch": 37.5,
       "eval_loss": 3.0010480880737305,
-      "eval_runtime": 2.9312,
-      "eval_samples_per_second": 704.488,
-      "eval_steps_per_second": 44.35,
+      "eval_runtime": 3.0576,
+      "eval_samples_per_second": 675.356,
+      "eval_steps_per_second": 42.516,
       "step": 30000
     },
     {
@@ -187,17 +187,17 @@
     {
       "epoch": 40.0,
       "eval_loss": 2.850881338119507,
-      "eval_runtime": 3.0495,
-      "eval_samples_per_second": 677.15,
-      "eval_steps_per_second": 42.629,
+      "eval_runtime": 3.0048,
+      "eval_samples_per_second": 687.232,
+      "eval_steps_per_second": 43.264,
       "step": 32000
     },
     {
       "epoch": 42.5,
       "eval_loss": 2.733860731124878,
-      "eval_runtime": 2.9214,
-      "eval_samples_per_second": 706.846,
-      "eval_steps_per_second": 44.499,
+      "eval_runtime": 3.0203,
+      "eval_samples_per_second": 683.699,
+      "eval_steps_per_second": 43.042,
       "step": 34000
     },
     {
@@ -210,17 +210,17 @@
     {
       "epoch": 45.0,
       "eval_loss": 2.6182146072387695,
-      "eval_runtime": 3.223,
-      "eval_samples_per_second": 640.711,
-      "eval_steps_per_second": 40.335,
+      "eval_runtime": 3.0392,
+      "eval_samples_per_second": 679.464,
+      "eval_steps_per_second": 42.775,
       "step": 36000
     },
     {
       "epoch": 47.5,
       "eval_loss": 2.561305046081543,
-      "eval_runtime": 3.1317,
-      "eval_samples_per_second": 659.384,
-      "eval_steps_per_second": 41.511,
+      "eval_runtime": 2.9856,
+      "eval_samples_per_second": 691.642,
+      "eval_steps_per_second": 43.542,
       "step": 38000
     },
     {
@@ -233,17 +233,17 @@
     {
       "epoch": 50.0,
       "eval_loss": 2.4945614337921143,
-      "eval_runtime": 3.1343,
-      "eval_samples_per_second": 658.831,
-      "eval_steps_per_second": 41.476,
+      "eval_runtime": 3.0785,
+      "eval_samples_per_second": 670.787,
+      "eval_steps_per_second": 42.229,
       "step": 40000
     },
     {
       "epoch": 52.5,
       "eval_loss": 2.4196503162384033,
-      "eval_runtime": 3.0349,
-      "eval_samples_per_second": 680.411,
-      "eval_steps_per_second": 42.835,
+      "eval_runtime": 3.1293,
+      "eval_samples_per_second": 659.901,
+      "eval_steps_per_second": 41.543,
       "step": 42000
     },
     {
@@ -256,17 +256,17 @@
     {
       "epoch": 55.0,
       "eval_loss": 2.3687477111816406,
-      "eval_runtime": 3.0804,
-      "eval_samples_per_second": 670.376,
-      "eval_steps_per_second": 42.203,
+      "eval_runtime": 3.0418,
+      "eval_samples_per_second": 678.883,
+      "eval_steps_per_second": 42.738,
       "step": 44000
     },
     {
       "epoch": 57.5,
       "eval_loss": 2.2801854610443115,
-      "eval_runtime": 3.039,
-      "eval_samples_per_second": 679.507,
-      "eval_steps_per_second": 42.778,
+      "eval_runtime": 2.562,
+      "eval_samples_per_second": 805.998,
+      "eval_steps_per_second": 50.741,
       "step": 46000
     },
     {
@@ -279,17 +279,17 @@
     {
       "epoch": 60.0,
       "eval_loss": 2.262115478515625,
-      "eval_runtime": 3.1233,
-      "eval_samples_per_second": 661.165,
-      "eval_steps_per_second": 41.623,
+      "eval_runtime": 1.9284,
+      "eval_samples_per_second": 1070.809,
+      "eval_steps_per_second": 67.412,
       "step": 48000
     },
     {
       "epoch": 62.5,
       "eval_loss": 2.2170372009277344,
-      "eval_runtime": 3.1043,
-      "eval_samples_per_second": 665.2,
-      "eval_steps_per_second": 41.877,
+      "eval_runtime": 1.6322,
+      "eval_samples_per_second": 1265.182,
+      "eval_steps_per_second": 79.648,
       "step": 50000
     },
     {
@@ -302,17 +302,17 @@
     {
       "epoch": 65.0,
       "eval_loss": 2.1907379627227783,
-      "eval_runtime": 3.0605,
-      "eval_samples_per_second": 674.72,
-      "eval_steps_per_second": 42.476,
+      "eval_runtime": 1.7,
+      "eval_samples_per_second": 1214.702,
+      "eval_steps_per_second": 76.47,
       "step": 52000
     },
     {
       "epoch": 67.5,
       "eval_loss": 2.1659305095672607,
-      "eval_runtime": 3.11,
-      "eval_samples_per_second": 663.989,
-      "eval_steps_per_second": 41.801,
+      "eval_runtime": 1.8055,
+      "eval_samples_per_second": 1143.712,
+      "eval_steps_per_second": 72.001,
       "step": 54000
     },
     {
@@ -325,17 +325,17 @@
     {
       "epoch": 70.0,
       "eval_loss": 2.127293825149536,
-      "eval_runtime": 3.0456,
-      "eval_samples_per_second": 678.031,
-      "eval_steps_per_second": 42.685,
+      "eval_runtime": 1.7543,
+      "eval_samples_per_second": 1177.082,
+      "eval_steps_per_second": 74.102,
       "step": 56000
     },
     {
       "epoch": 72.5,
       "eval_loss": 2.087448835372925,
-      "eval_runtime": 3.052,
-      "eval_samples_per_second": 676.616,
-      "eval_steps_per_second": 42.596,
+      "eval_runtime": 1.7243,
+      "eval_samples_per_second": 1197.568,
+      "eval_steps_per_second": 75.392,
       "step": 58000
     },
     {
@@ -348,17 +348,17 @@
     {
       "epoch": 75.0,
       "eval_loss": 2.0742504596710205,
-      "eval_runtime": 3.3514,
-      "eval_samples_per_second": 616.163,
-      "eval_steps_per_second": 38.79,
+      "eval_runtime": 1.7269,
+      "eval_samples_per_second": 1195.792,
+      "eval_steps_per_second": 75.28,
       "step": 60000
     },
     {
       "epoch": 77.5,
       "eval_loss": 2.0582902431488037,
-      "eval_runtime": 2.6814,
-      "eval_samples_per_second": 770.12,
-      "eval_steps_per_second": 48.482,
+      "eval_runtime": 1.7674,
+      "eval_samples_per_second": 1168.411,
+      "eval_steps_per_second": 73.556,
       "step": 62000
     },
     {
@@ -371,17 +371,17 @@
     {
       "epoch": 80.0,
       "eval_loss": 2.037095785140991,
-      "eval_runtime": 1.6551,
-      "eval_samples_per_second": 1247.667,
-      "eval_steps_per_second": 78.546,
+      "eval_runtime": 1.6339,
+      "eval_samples_per_second": 1263.872,
+      "eval_steps_per_second": 79.566,
       "step": 64000
     },
     {
       "epoch": 82.5,
       "eval_loss": 2.0038888454437256,
-      "eval_runtime": 1.7042,
-      "eval_samples_per_second": 1211.685,
-      "eval_steps_per_second": 76.28,
+      "eval_runtime": 1.6907,
+      "eval_samples_per_second": 1221.38,
+      "eval_steps_per_second": 76.891,
       "step": 66000
     },
     {
@@ -394,17 +394,17 @@
     {
       "epoch": 85.0,
       "eval_loss": 1.9900726079940796,
-      "eval_runtime": 1.872,
-      "eval_samples_per_second": 1103.097,
-      "eval_steps_per_second": 69.444,
+      "eval_runtime": 1.6879,
+      "eval_samples_per_second": 1223.403,
+      "eval_steps_per_second": 77.018,
       "step": 68000
     },
     {
       "epoch": 87.5,
       "eval_loss": 1.9754005670547485,
-      "eval_runtime": 1.6651,
-      "eval_samples_per_second": 1240.19,
-      "eval_steps_per_second": 78.075,
+      "eval_runtime": 1.7314,
+      "eval_samples_per_second": 1192.693,
+      "eval_steps_per_second": 75.085,
       "step": 70000
     },
     {
@@ -417,17 +417,17 @@
     {
       "epoch": 90.0,
       "eval_loss": 1.9565609693527222,
-      "eval_runtime": 1.6688,
-      "eval_samples_per_second": 1237.389,
-      "eval_steps_per_second": 77.899,
+      "eval_runtime": 1.8855,
+      "eval_samples_per_second": 1095.221,
+      "eval_steps_per_second": 68.949,
       "step": 72000
     },
     {
       "epoch": 92.5,
       "eval_loss": 1.940429449081421,
-      "eval_runtime": 1.67,
-      "eval_samples_per_second": 1236.495,
-      "eval_steps_per_second": 77.842,
+      "eval_runtime": 1.7766,
+      "eval_samples_per_second": 1162.312,
+      "eval_steps_per_second": 73.172,
       "step": 74000
     },
     {
@@ -440,17 +440,17 @@
     {
       "epoch": 95.0,
       "eval_loss": 1.9128267765045166,
-      "eval_runtime": 1.638,
-      "eval_samples_per_second": 1260.669,
-      "eval_steps_per_second": 79.364,
+      "eval_runtime": 1.8509,
+      "eval_samples_per_second": 1115.674,
+      "eval_steps_per_second": 70.236,
       "step": 76000
     },
     {
       "epoch": 97.5,
       "eval_loss": 1.939635157585144,
-      "eval_runtime": 1.7065,
-      "eval_samples_per_second": 1210.111,
-      "eval_steps_per_second": 76.181,
+      "eval_runtime": 1.7798,
+      "eval_samples_per_second": 1160.26,
+      "eval_steps_per_second": 73.043,
       "step": 78000
     },
     {
@@ -463,17 +463,17 @@
     {
       "epoch": 100.0,
       "eval_loss": 1.959123134613037,
-      "eval_runtime": 1.7328,
-      "eval_samples_per_second": 1191.745,
-      "eval_steps_per_second": 75.025,
+      "eval_runtime": 1.6637,
+      "eval_samples_per_second": 1241.231,
+      "eval_steps_per_second": 78.14,
       "step": 80000
     },
     {
       "epoch": 102.5,
       "eval_loss": 1.9078463315963745,
-      "eval_runtime": 1.6402,
-      "eval_samples_per_second": 1258.997,
-      "eval_steps_per_second": 79.259,
+      "eval_runtime": 1.6559,
+      "eval_samples_per_second": 1247.034,
+      "eval_steps_per_second": 78.506,
       "step": 82000
     },
     {
@@ -486,17 +486,17 @@
     {
       "epoch": 105.0,
       "eval_loss": 1.9178215265274048,
-      "eval_runtime": 1.5958,
-      "eval_samples_per_second": 1293.982,
-      "eval_steps_per_second": 81.461,
+      "eval_runtime": 1.7131,
+      "eval_samples_per_second": 1205.385,
+      "eval_steps_per_second": 75.884,
       "step": 84000
     },
     {
       "epoch": 107.5,
       "eval_loss": 1.9046084880828857,
-      "eval_runtime": 3.0454,
-      "eval_samples_per_second": 678.079,
-      "eval_steps_per_second": 42.688,
+      "eval_runtime": 1.7051,
+      "eval_samples_per_second": 1211.05,
+      "eval_steps_per_second": 76.24,
       "step": 86000
     },
     {
@@ -509,9 +509,9 @@
     {
       "epoch": 110.0,
       "eval_loss": 1.8918195962905884,
-      "eval_runtime": 3.034,
-      "eval_samples_per_second": 680.628,
-      "eval_steps_per_second": 42.848,
+      "eval_runtime": 1.7296,
+      "eval_samples_per_second": 1193.906,
+      "eval_steps_per_second": 75.161,
       "step": 88000
     }
   ],