Training in progress, step 140, checkpoint

checkpoint-140/adapter_config.json

@@ -28,16 +28,16 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "fc1",
-    "up_proj",
     "out_proj",
+    "fc1",
     "v_proj",
+    "q_proj",
     "k_proj",
-    "fc2",
-    "o_proj",
     "gate_proj",
-    "q_proj",
-    "down_proj"
+    "fc2",
+    "down_proj",
+    "up_proj",
+    "o_proj"
   ],
   "task_type": "CAUSAL_LM",
   "trainable_token_indices": null,

checkpoint-140/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5bebfa0a3690708d41785bbf654a92bc44456c31eca56a100b968208e0f1c7d1
+oid sha256:549ad296ebfaff6156895999d6bf17d16f2282b8c4e2684525a674d0211a46be
 size 2839126480

checkpoint-140/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c5d291b36748b1ec1acc94304fecc2dcc8c032a533ec4abbf2630b5929852837
+oid sha256:ac4e15c11d602c0abc4f586e2d4edb042836ca32f4c851243e6aae2068ff5fe8
 size 5678690152

checkpoint-140/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1b9e544c7c9f0c7141a340fb780b7b8d9b875d82948159f9e783f4e56460ff3b
+oid sha256:6e594e01bfb1472dcf3e3e35010234d2fda69d99c079c1d82240887066ba01c2
 size 14244

checkpoint-140/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b77485ffb044d9521406332dff11a4a245371dfc5223ef3de541f5b483d4b86e
+oid sha256:ba4beee13c42a892f60a7b108ff31c7a6a3447e1c74a5da749becbd8d038ccef
 size 1064

checkpoint-140/trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.6730769230769231,
+  "epoch": 3.6013071895424837,
   "eval_steps": 10,
   "global_step": 140,
   "is_hyper_param_search": false,
@@ -10,276 +10,276 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.04807692307692308,
-      "grad_norm": 19.535831451416016,
-      "learning_rate": 0.00019800995024875625,
-      "loss": 15.9815,
-      "mean_token_accuracy": 0.6259999990463256,
+      "epoch": 0.26143790849673204,
+      "grad_norm": 80.0328369140625,
+      "learning_rate": 1.9682539682539684e-05,
+      "loss": 22.6724,
+      "mean_token_accuracy": 0.49049999862909316,
       "num_tokens": 49120.0,
       "step": 10
     },
     {
-      "epoch": 0.04807692307692308,
-      "eval_loss": 1.2629932165145874,
-      "eval_mean_token_accuracy": 0.8946153659086961,
+      "epoch": 0.26143790849673204,
+      "eval_loss": 4.981190204620361,
+      "eval_mean_token_accuracy": 0.5223076802033645,
       "eval_num_tokens": 49120.0,
-      "eval_runtime": 17.4778,
-      "eval_samples_per_second": 2.861,
-      "eval_steps_per_second": 0.744,
+      "eval_runtime": 17.5192,
+      "eval_samples_per_second": 2.854,
+      "eval_steps_per_second": 0.742,
       "step": 10
     },
     {
-      "epoch": 0.09615384615384616,
-      "grad_norm": 7.8091559410095215,
-      "learning_rate": 0.00018805970149253734,
-      "loss": 2.5529,
-      "mean_token_accuracy": 0.9347499996423722,
+      "epoch": 0.5228758169934641,
+      "grad_norm": 57.1182746887207,
+      "learning_rate": 1.8624338624338625e-05,
+      "loss": 17.5658,
+      "mean_token_accuracy": 0.5591249987483025,
       "num_tokens": 98240.0,
       "step": 20
     },
     {
-      "epoch": 0.09615384615384616,
-      "eval_loss": 0.14167079329490662,
-      "eval_mean_token_accuracy": 0.9773077139487634,
+      "epoch": 0.5228758169934641,
+      "eval_loss": 3.730682134628296,
+      "eval_mean_token_accuracy": 0.5742307580434359,
       "eval_num_tokens": 98240.0,
-      "eval_runtime": 17.4221,
-      "eval_samples_per_second": 2.87,
-      "eval_steps_per_second": 0.746,
+      "eval_runtime": 17.4511,
+      "eval_samples_per_second": 2.865,
+      "eval_steps_per_second": 0.745,
       "step": 20
     },
     {
-      "epoch": 0.14423076923076922,
-      "grad_norm": 3.069196939468384,
-      "learning_rate": 0.00017810945273631843,
-      "loss": 0.3788,
-      "mean_token_accuracy": 0.9778750196099282,
+      "epoch": 0.7843137254901961,
+      "grad_norm": 58.24352264404297,
+      "learning_rate": 1.7566137566137566e-05,
+      "loss": 13.113,
+      "mean_token_accuracy": 0.6388749912381172,
       "num_tokens": 147360.0,
       "step": 30
     },
     {
-      "epoch": 0.14423076923076922,
-      "eval_loss": 0.03845745697617531,
-      "eval_mean_token_accuracy": 0.9788461740200336,
+      "epoch": 0.7843137254901961,
+      "eval_loss": 2.7529046535491943,
+      "eval_mean_token_accuracy": 0.7115384431985708,
       "eval_num_tokens": 147360.0,
-      "eval_runtime": 17.5754,
-      "eval_samples_per_second": 2.845,
-      "eval_steps_per_second": 0.74,
+      "eval_runtime": 17.6786,
+      "eval_samples_per_second": 2.828,
+      "eval_steps_per_second": 0.735,
       "step": 30
     },
     {
-      "epoch": 0.19230769230769232,
-      "grad_norm": 6.14934778213501,
-      "learning_rate": 0.0001681592039800995,
-      "loss": 0.1621,
-      "mean_token_accuracy": 0.9760000124573708,
-      "num_tokens": 196480.0,
+      "epoch": 1.026143790849673,
+      "grad_norm": 30.73020362854004,
+      "learning_rate": 1.6507936507936507e-05,
+      "loss": 8.7369,
+      "mean_token_accuracy": 0.7555405417004147,
+      "num_tokens": 192796.0,
       "step": 40
     },
     {
-      "epoch": 0.19230769230769232,
-      "eval_loss": 0.03434291481971741,
-      "eval_mean_token_accuracy": 0.9788461740200336,
-      "eval_num_tokens": 196480.0,
-      "eval_runtime": 17.4843,
-      "eval_samples_per_second": 2.86,
-      "eval_steps_per_second": 0.744,
+      "epoch": 1.026143790849673,
+      "eval_loss": 1.836286187171936,
+      "eval_mean_token_accuracy": 0.8476923199800345,
+      "eval_num_tokens": 192796.0,
+      "eval_runtime": 17.4538,
+      "eval_samples_per_second": 2.865,
+      "eval_steps_per_second": 0.745,
       "step": 40
     },
     {
-      "epoch": 0.2403846153846154,
-      "grad_norm": 7.388150691986084,
-      "learning_rate": 0.00015820895522388059,
-      "loss": 0.1563,
-      "mean_token_accuracy": 0.9780000209808349,
-      "num_tokens": 245600.0,
+      "epoch": 1.287581699346405,
+      "grad_norm": 21.440105438232422,
+      "learning_rate": 1.544973544973545e-05,
+      "loss": 6.2989,
+      "mean_token_accuracy": 0.8789999932050705,
+      "num_tokens": 241916.0,
       "step": 50
     },
     {
-      "epoch": 0.2403846153846154,
-      "eval_loss": 0.05149823799729347,
-      "eval_mean_token_accuracy": 0.9788461740200336,
-      "eval_num_tokens": 245600.0,
-      "eval_runtime": 17.5208,
-      "eval_samples_per_second": 2.854,
-      "eval_steps_per_second": 0.742,
+      "epoch": 1.287581699346405,
+      "eval_loss": 1.3245856761932373,
+      "eval_mean_token_accuracy": 0.8942307508908786,
+      "eval_num_tokens": 241916.0,
+      "eval_runtime": 17.6384,
+      "eval_samples_per_second": 2.835,
+      "eval_steps_per_second": 0.737,
       "step": 50
     },
     {
-      "epoch": 0.28846153846153844,
-      "grad_norm": 5.478137969970703,
-      "learning_rate": 0.00014825870646766168,
-      "loss": 0.177,
-      "mean_token_accuracy": 0.9782500207424164,
-      "num_tokens": 294720.0,
+      "epoch": 1.5490196078431373,
+      "grad_norm": 10.908184051513672,
+      "learning_rate": 1.4391534391534392e-05,
+      "loss": 4.5427,
+      "mean_token_accuracy": 0.9057500049471855,
+      "num_tokens": 291036.0,
       "step": 60
     },
     {
-      "epoch": 0.28846153846153844,
-      "eval_loss": 0.03306853771209717,
-      "eval_mean_token_accuracy": 0.9788461740200336,
-      "eval_num_tokens": 294720.0,
-      "eval_runtime": 17.4723,
+      "epoch": 1.5490196078431373,
+      "eval_loss": 0.9362902045249939,
+      "eval_mean_token_accuracy": 0.9126923221808213,
+      "eval_num_tokens": 291036.0,
+      "eval_runtime": 17.472,
       "eval_samples_per_second": 2.862,
       "eval_steps_per_second": 0.744,
       "step": 60
     },
     {
-      "epoch": 0.33653846153846156,
-      "grad_norm": 4.636706352233887,
-      "learning_rate": 0.0001383084577114428,
-      "loss": 0.1614,
-      "mean_token_accuracy": 0.9776250213384629,
-      "num_tokens": 343840.0,
+      "epoch": 1.8104575163398693,
+      "grad_norm": 11.274917602539062,
+      "learning_rate": 1.3333333333333333e-05,
+      "loss": 3.4538,
+      "mean_token_accuracy": 0.9126250177621842,
+      "num_tokens": 340156.0,
       "step": 70
     },
     {
-      "epoch": 0.33653846153846156,
-      "eval_loss": 0.041035111993551254,
-      "eval_mean_token_accuracy": 0.9788461740200336,
-      "eval_num_tokens": 343840.0,
-      "eval_runtime": 17.567,
-      "eval_samples_per_second": 2.846,
-      "eval_steps_per_second": 0.74,
+      "epoch": 1.8104575163398693,
+      "eval_loss": 0.7567419409751892,
+      "eval_mean_token_accuracy": 0.9103846183189979,
+      "eval_num_tokens": 340156.0,
+      "eval_runtime": 17.5544,
+      "eval_samples_per_second": 2.848,
+      "eval_steps_per_second": 0.741,
       "step": 70
     },
     {
-      "epoch": 0.38461538461538464,
-      "grad_norm": 1.8013979196548462,
-      "learning_rate": 0.00012835820895522389,
-      "loss": 0.1449,
-      "mean_token_accuracy": 0.9785000205039978,
-      "num_tokens": 392960.0,
+      "epoch": 2.052287581699346,
+      "grad_norm": 13.183298110961914,
+      "learning_rate": 1.2275132275132276e-05,
+      "loss": 2.5315,
+      "mean_token_accuracy": 0.9293243321212562,
+      "num_tokens": 385592.0,
       "step": 80
     },
     {
-      "epoch": 0.38461538461538464,
-      "eval_loss": 0.033909473568201065,
-      "eval_mean_token_accuracy": 0.9788461740200336,
-      "eval_num_tokens": 392960.0,
-      "eval_runtime": 17.4599,
-      "eval_samples_per_second": 2.864,
-      "eval_steps_per_second": 0.745,
+      "epoch": 2.052287581699346,
+      "eval_loss": 0.6147329807281494,
+      "eval_mean_token_accuracy": 0.9292307771169223,
+      "eval_num_tokens": 385592.0,
+      "eval_runtime": 17.582,
+      "eval_samples_per_second": 2.844,
+      "eval_steps_per_second": 0.739,
       "step": 80
     },
     {
-      "epoch": 0.4326923076923077,
-      "grad_norm": 0.7925957441329956,
-      "learning_rate": 0.00011840796019900498,
-      "loss": 0.1343,
-      "mean_token_accuracy": 0.979250019788742,
-      "num_tokens": 442080.0,
+      "epoch": 2.313725490196078,
+      "grad_norm": 12.541313171386719,
+      "learning_rate": 1.1216931216931217e-05,
+      "loss": 2.2552,
+      "mean_token_accuracy": 0.9353750020265579,
+      "num_tokens": 434712.0,
       "step": 90
     },
     {
-      "epoch": 0.4326923076923077,
-      "eval_loss": 0.03350621089339256,
-      "eval_mean_token_accuracy": 0.9788461740200336,
-      "eval_num_tokens": 442080.0,
-      "eval_runtime": 17.5312,
-      "eval_samples_per_second": 2.852,
-      "eval_steps_per_second": 0.742,
+      "epoch": 2.313725490196078,
+      "eval_loss": 0.5065863132476807,
+      "eval_mean_token_accuracy": 0.9465384529187129,
+      "eval_num_tokens": 434712.0,
+      "eval_runtime": 17.6818,
+      "eval_samples_per_second": 2.828,
+      "eval_steps_per_second": 0.735,
       "step": 90
     },
     {
-      "epoch": 0.4807692307692308,
-      "grad_norm": 3.3666160106658936,
-      "learning_rate": 0.00010845771144278607,
-      "loss": 0.1484,
-      "mean_token_accuracy": 0.9778750211000442,
-      "num_tokens": 491200.0,
+      "epoch": 2.57516339869281,
+      "grad_norm": 7.007925033569336,
+      "learning_rate": 1.015873015873016e-05,
+      "loss": 1.9547,
+      "mean_token_accuracy": 0.9474999904632568,
+      "num_tokens": 483832.0,
       "step": 100
     },
     {
-      "epoch": 0.4807692307692308,
-      "eval_loss": 0.03422728180885315,
-      "eval_mean_token_accuracy": 0.9788461740200336,
-      "eval_num_tokens": 491200.0,
-      "eval_runtime": 17.4416,
-      "eval_samples_per_second": 2.867,
-      "eval_steps_per_second": 0.745,
+      "epoch": 2.57516339869281,
+      "eval_loss": 0.4519544243812561,
+      "eval_mean_token_accuracy": 0.9511538331325238,
+      "eval_num_tokens": 483832.0,
+      "eval_runtime": 17.5399,
+      "eval_samples_per_second": 2.851,
+      "eval_steps_per_second": 0.741,
       "step": 100
     },
     {
-      "epoch": 0.5288461538461539,
-      "grad_norm": 1.1528574228286743,
-      "learning_rate": 9.850746268656717e-05,
-      "loss": 0.1372,
-      "mean_token_accuracy": 0.9783750206232071,
-      "num_tokens": 540320.0,
+      "epoch": 2.8366013071895426,
+      "grad_norm": 6.267516613006592,
+      "learning_rate": 9.1005291005291e-06,
+      "loss": 1.7199,
+      "mean_token_accuracy": 0.9503749877214431,
+      "num_tokens": 532952.0,
       "step": 110
     },
     {
-      "epoch": 0.5288461538461539,
-      "eval_loss": 0.03267410025000572,
-      "eval_mean_token_accuracy": 0.9788461740200336,
-      "eval_num_tokens": 540320.0,
-      "eval_runtime": 17.6379,
-      "eval_samples_per_second": 2.835,
-      "eval_steps_per_second": 0.737,
+      "epoch": 2.8366013071895426,
+      "eval_loss": 0.396070659160614,
+      "eval_mean_token_accuracy": 0.9519230631681589,
+      "eval_num_tokens": 532952.0,
+      "eval_runtime": 17.6107,
+      "eval_samples_per_second": 2.839,
+      "eval_steps_per_second": 0.738,
       "step": 110
     },
     {
-      "epoch": 0.5769230769230769,
-      "grad_norm": 1.7770888805389404,
-      "learning_rate": 8.855721393034826e-05,
-      "loss": 0.1499,
-      "mean_token_accuracy": 0.9773750215768814,
-      "num_tokens": 589440.0,
+      "epoch": 3.0784313725490198,
+      "grad_norm": 6.098326683044434,
+      "learning_rate": 8.042328042328043e-06,
+      "loss": 1.3242,
+      "mean_token_accuracy": 0.9683783892038706,
+      "num_tokens": 578388.0,
       "step": 120
     },
     {
-      "epoch": 0.5769230769230769,
-      "eval_loss": 0.03415742516517639,
-      "eval_mean_token_accuracy": 0.9788461740200336,
-      "eval_num_tokens": 589440.0,
-      "eval_runtime": 17.4268,
-      "eval_samples_per_second": 2.869,
-      "eval_steps_per_second": 0.746,
+      "epoch": 3.0784313725490198,
+      "eval_loss": 0.3188876807689667,
+      "eval_mean_token_accuracy": 0.9742307938062228,
+      "eval_num_tokens": 578388.0,
+      "eval_runtime": 17.8412,
+      "eval_samples_per_second": 2.802,
+      "eval_steps_per_second": 0.729,
       "step": 120
     },
     {
-      "epoch": 0.625,
-      "grad_norm": 0.9424811601638794,
-      "learning_rate": 7.860696517412935e-05,
-      "loss": 0.1354,
-      "mean_token_accuracy": 0.9777500197291374,
-      "num_tokens": 638560.0,
+      "epoch": 3.3398692810457518,
+      "grad_norm": 9.010005950927734,
+      "learning_rate": 6.984126984126984e-06,
+      "loss": 1.1048,
+      "mean_token_accuracy": 0.9745000153779984,
+      "num_tokens": 627508.0,
       "step": 130
     },
     {
-      "epoch": 0.625,
-      "eval_loss": 0.03265358507633209,
-      "eval_mean_token_accuracy": 0.9788461740200336,
-      "eval_num_tokens": 638560.0,
-      "eval_runtime": 17.6065,
-      "eval_samples_per_second": 2.84,
-      "eval_steps_per_second": 0.738,
+      "epoch": 3.3398692810457518,
+      "eval_loss": 0.2367120385169983,
+      "eval_mean_token_accuracy": 0.9719230899443994,
+      "eval_num_tokens": 627508.0,
+      "eval_runtime": 17.8266,
+      "eval_samples_per_second": 2.805,
+      "eval_steps_per_second": 0.729,
       "step": 130
     },
     {
-      "epoch": 0.6730769230769231,
-      "grad_norm": 2.114447593688965,
-      "learning_rate": 6.865671641791044e-05,
-      "loss": 0.1343,
-      "mean_token_accuracy": 0.97887502014637,
-      "num_tokens": 687680.0,
+      "epoch": 3.6013071895424837,
+      "grad_norm": 8.258376121520996,
+      "learning_rate": 5.925925925925926e-06,
+      "loss": 0.7666,
+      "mean_token_accuracy": 0.9731250181794167,
+      "num_tokens": 676628.0,
       "step": 140
     },
     {
-      "epoch": 0.6730769230769231,
-      "eval_loss": 0.033645644783973694,
-      "eval_mean_token_accuracy": 0.9788461740200336,
-      "eval_num_tokens": 687680.0,
-      "eval_runtime": 17.4491,
-      "eval_samples_per_second": 2.865,
-      "eval_steps_per_second": 0.745,
+      "epoch": 3.6013071895424837,
+      "eval_loss": 0.14354808628559113,
+      "eval_mean_token_accuracy": 0.9723077049622169,
+      "eval_num_tokens": 676628.0,
+      "eval_runtime": 17.4766,
+      "eval_samples_per_second": 2.861,
+      "eval_steps_per_second": 0.744,
       "step": 140
     }
   ],
   "logging_steps": 10,
-  "max_steps": 208,
+  "max_steps": 195,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 1,
+  "num_train_epochs": 5,
   "save_steps": 20,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -293,7 +293,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1.788184758936576e+16,
+  "total_flos": 1.7594460753108096e+16,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null

checkpoint-140/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:09a9baff00283695ce70ca75d7d996c90cc5734afa7d9ed3f9a6113fde47b709
+oid sha256:7fa3299c8cd6826c610bec43e932e8377d63e21789add24e4c875f4d75cb544d
 size 5880