bugfix in loading and data processing

model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fd5067b43ff84e6568feb148b9dada8aa0608d516364fb3ae6268dc6a238f160
+oid sha256:9a55f92b3ec1d5cc83a8e5e5ebb644b016c89d6672f89611c730144c9d8e23db
 size 4943274328

model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:35b9981cf2af1a882cd5baf8b395d15cc69814898586d1330e4987837cb6ed4e
+oid sha256:81801955b717ee7af3e03d0af29b78dcf85765b62a0f59765e1425fa418f51bf
 size 1050673280

model.safetensors.index.json

@@ -3,7 +3,6 @@
     "total_size": 5993930752
   },
   "weight_map": {
-    "lm_head.weight": "model-00002-of-00002.safetensors",
     "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
     "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",

trainer_state.json

@@ -10,208 +10,208 @@
   "log_history": [
     {
       "epoch": 0.10256410256410256,
-      "grad_norm": 5.980730056762695,
+      "grad_norm": 6.330384731292725,
       "learning_rate": 2e-05,
-      "loss": 0.9641,
+      "loss": 0.9917,
       "step": 5
     },
     {
       "epoch": 0.20512820512820512,
-      "grad_norm": 4.265922546386719,
+      "grad_norm": 2.223167896270752,
       "learning_rate": 1.9936215093023884e-05,
-      "loss": 0.7271,
+      "loss": 0.7004,
       "step": 10
     },
     {
       "epoch": 0.3076923076923077,
-      "grad_norm": 1.7026264667510986,
+      "grad_norm": 1.8093996047973633,
       "learning_rate": 1.974567407496712e-05,
-      "loss": 0.5734,
+      "loss": 0.5531,
       "step": 15
     },
     {
       "epoch": 0.41025641025641024,
-      "grad_norm": 1.6455885171890259,
+      "grad_norm": 1.6542998552322388,
       "learning_rate": 1.9430807674052092e-05,
-      "loss": 0.4955,
+      "loss": 0.4873,
       "step": 20
     },
     {
       "epoch": 0.5128205128205128,
-      "grad_norm": 1.5250736474990845,
+      "grad_norm": 1.5152506828308105,
       "learning_rate": 1.899563263509725e-05,
-      "loss": 0.4722,
+      "loss": 0.4658,
       "step": 25
     },
     {
       "epoch": 0.6153846153846154,
-      "grad_norm": 1.467634916305542,
+      "grad_norm": 1.466179370880127,
       "learning_rate": 1.8445700477978207e-05,
-      "loss": 0.4283,
+      "loss": 0.4239,
       "step": 30
     },
     {
       "epoch": 0.717948717948718,
-      "grad_norm": 1.3555026054382324,
+      "grad_norm": 1.3292107582092285,
       "learning_rate": 1.778802667699196e-05,
-      "loss": 0.417,
+      "loss": 0.4138,
       "step": 35
     },
     {
       "epoch": 0.8205128205128205,
-      "grad_norm": 1.4207082986831665,
+      "grad_norm": 1.4440351724624634,
       "learning_rate": 1.7031001164581828e-05,
-      "loss": 0.3969,
+      "loss": 0.3948,
       "step": 40
     },
     {
       "epoch": 0.9230769230769231,
-      "grad_norm": 1.3894375562667847,
+      "grad_norm": 1.3163414001464844,
       "learning_rate": 1.618428130112533e-05,
-      "loss": 0.3896,
+      "loss": 0.3881,
       "step": 45
     },
     {
       "epoch": 1.0205128205128204,
-      "grad_norm": 1.3288708925247192,
+      "grad_norm": 1.3107187747955322,
       "learning_rate": 1.5258668676167548e-05,
-      "loss": 0.3586,
+      "loss": 0.3565,
       "step": 50
     },
     {
       "epoch": 1.123076923076923,
-      "grad_norm": 1.3780906200408936,
+      "grad_norm": 1.3364475965499878,
       "learning_rate": 1.4265971312744252e-05,
-      "loss": 0.3267,
+      "loss": 0.324,
       "step": 55
     },
     {
       "epoch": 1.2256410256410257,
-      "grad_norm": 1.1098214387893677,
+      "grad_norm": 1.1268199682235718,
       "learning_rate": 1.3218853032651719e-05,
-      "loss": 0.3194,
+      "loss": 0.3167,
       "step": 60
     },
     {
       "epoch": 1.3282051282051281,
-      "grad_norm": 1.3044273853302002,
+      "grad_norm": 1.2984614372253418,
       "learning_rate": 1.2130671904307692e-05,
-      "loss": 0.3065,
+      "loss": 0.3045,
       "step": 65
     },
     {
       "epoch": 1.4307692307692308,
-      "grad_norm": 1.2135035991668701,
+      "grad_norm": 1.2189068794250488,
       "learning_rate": 1.1015309834121083e-05,
-      "loss": 0.2907,
+      "loss": 0.2888,
       "step": 70
     },
     {
       "epoch": 1.5333333333333332,
-      "grad_norm": 1.172560214996338,
+      "grad_norm": 1.208184003829956,
       "learning_rate": 9.886995475270205e-06,
-      "loss": 0.299,
+      "loss": 0.2973,
       "step": 75
     },
     {
       "epoch": 1.6358974358974359,
-      "grad_norm": 1.200344204902649,
+      "grad_norm": 1.1965588331222534,
       "learning_rate": 8.76012271303888e-06,
-      "loss": 0.2953,
+      "loss": 0.2931,
       "step": 80
     },
     {
       "epoch": 1.7384615384615385,
-      "grad_norm": 1.1982589960098267,
+      "grad_norm": 1.1966915130615234,
       "learning_rate": 7.649067042289681e-06,
-      "loss": 0.2942,
+      "loss": 0.2915,
       "step": 85
     },
     {
       "epoch": 1.8410256410256411,
-      "grad_norm": 1.2551528215408325,
+      "grad_norm": 1.2417312860488892,
       "learning_rate": 6.568002179543409e-06,
-      "loss": 0.2883,
+      "loss": 0.2858,
       "step": 90
     },
     {
       "epoch": 1.9435897435897436,
-      "grad_norm": 1.276993989944458,
+      "grad_norm": 1.2801405191421509,
       "learning_rate": 5.530719249141148e-06,
-      "loss": 0.2885,
+      "loss": 0.287,
       "step": 95
     },
     {
       "epoch": 2.041025641025641,
-      "grad_norm": 1.176174521446228,
+      "grad_norm": 1.1753469705581665,
       "learning_rate": 4.550450850127626e-06,
-      "loss": 0.2587,
+      "loss": 0.2573,
       "step": 100
     },
     {
       "epoch": 2.1435897435897435,
-      "grad_norm": 1.3641650676727295,
+      "grad_norm": 1.3422598838806152,
       "learning_rate": 3.6397022482313804e-06,
-      "loss": 0.2231,
+      "loss": 0.2207,
       "step": 105
     },
     {
       "epoch": 2.246153846153846,
-      "grad_norm": 1.1558363437652588,
+      "grad_norm": 1.1634305715560913,
       "learning_rate": 2.8100918464225304e-06,
-      "loss": 0.2215,
+      "loss": 0.2188,
       "step": 110
     },
     {
       "epoch": 2.348717948717949,
-      "grad_norm": 1.2226226329803467,
+      "grad_norm": 1.2047914266586304,
       "learning_rate": 2.072202969162234e-06,
-      "loss": 0.2165,
+      "loss": 0.2144,
       "step": 115
     },
     {
       "epoch": 2.4512820512820515,
-      "grad_norm": 1.1804394721984863,
+      "grad_norm": 1.1728473901748657,
       "learning_rate": 1.4354488511294418e-06,
-      "loss": 0.2089,
+      "loss": 0.2065,
       "step": 120
     },
     {
       "epoch": 2.5538461538461537,
-      "grad_norm": 1.253951072692871,
+      "grad_norm": 1.2496421337127686,
       "learning_rate": 9.079525527612321e-07,
-      "loss": 0.2123,
+      "loss": 0.2107,
       "step": 125
     },
     {
       "epoch": 2.6564102564102563,
-      "grad_norm": 1.2056702375411987,
+      "grad_norm": 1.200203537940979,
       "learning_rate": 4.964433345219354e-07,
-      "loss": 0.2107,
+      "loss": 0.2083,
       "step": 130
     },
     {
       "epoch": 2.758974358974359,
-      "grad_norm": 1.1851439476013184,
+      "grad_norm": 1.2024612426757812,
       "learning_rate": 2.0617081185259512e-07,
-      "loss": 0.2129,
+      "loss": 0.2111,
       "step": 135
     },
     {
       "epoch": 2.8615384615384616,
-      "grad_norm": 1.1076850891113281,
+      "grad_norm": 1.1126112937927246,
       "learning_rate": 4.083798592444899e-08,
-      "loss": 0.2175,
+      "loss": 0.2157,
       "step": 140
     },
     {
       "epoch": 2.943589743589744,
       "step": 144,
       "total_flos": 5.491122506196582e+16,
-      "train_loss": 0.35015756347113186,
-      "train_runtime": 1479.6793,
-      "train_samples_per_second": 6.324,
-      "train_steps_per_second": 0.097
+      "train_loss": 0.34712929568356937,
+      "train_runtime": 1463.7078,
+      "train_samples_per_second": 6.393,
+      "train_steps_per_second": 0.098
     }
   ],
   "logging_steps": 5,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8ca955ae44219660913cf31e0416e5dada732a8cc00f42076a6739b75ba27b10
+oid sha256:10d62da16287a3bdc5aa2c1ba62da1ca7cc3a2218b5694c72c5959ee56c6cd93
 size 5624