Training in progress, step 100, checkpoint

checkpoint-100/adapter_config.json

@@ -28,16 +28,16 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "fc1",
-    "up_proj",
     "out_proj",
+    "fc1",
     "v_proj",
+    "q_proj",
     "k_proj",
-    "fc2",
-    "o_proj",
     "gate_proj",
-    "q_proj",
-    "down_proj"
+    "fc2",
+    "down_proj",
+    "up_proj",
+    "o_proj"
   ],
   "task_type": "CAUSAL_LM",
   "trainable_token_indices": null,

checkpoint-100/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3219ff424619f73e88d1a7ab31d4bf717362c88632b67e564523fccf14b6700b
+oid sha256:97bb7902edd622058ec9b3e1ba6af00ddf0f38798eddf41d703f2da96b024e06
 size 2839126480

checkpoint-100/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b7e64d9f25a395188487c0a873facea913061e7017d2f74eb242b06ff7e40386
+oid sha256:17e363ca06bcb24c36371e04f9a577253824f17c23299a9c67ce2732e5fadef3
 size 5678690152

checkpoint-100/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dad12f6c63b70b4aa3e59cacb7050751da9e1ffb089ed0b00a685cb0f0f1c682
+oid sha256:982d52cbfb794dabe573a4b81987c973c3c89987096944f0d45566faeae8f700
 size 14244

checkpoint-100/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ba48efa9dcbb02965f728f536956fdd49f8ec1aa0c93cb087aaa4c3ed9a30a88
+oid sha256:e12e31dd017e9524250842dde0204c7f228679a93737c9b081a9b448744f4bdb
 size 1064

checkpoint-100/trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.4807692307692308,
+  "epoch": 2.57516339869281,
   "eval_steps": 10,
   "global_step": 100,
   "is_hyper_param_search": false,
@@ -10,200 +10,200 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.04807692307692308,
-      "grad_norm": 19.535831451416016,
-      "learning_rate": 0.00019800995024875625,
-      "loss": 15.9815,
-      "mean_token_accuracy": 0.6259999990463256,
+      "epoch": 0.26143790849673204,
+      "grad_norm": 80.0328369140625,
+      "learning_rate": 1.9682539682539684e-05,
+      "loss": 22.6724,
+      "mean_token_accuracy": 0.49049999862909316,
       "num_tokens": 49120.0,
       "step": 10
     },
     {
-      "epoch": 0.04807692307692308,
-      "eval_loss": 1.2629932165145874,
-      "eval_mean_token_accuracy": 0.8946153659086961,
+      "epoch": 0.26143790849673204,
+      "eval_loss": 4.981190204620361,
+      "eval_mean_token_accuracy": 0.5223076802033645,
       "eval_num_tokens": 49120.0,
-      "eval_runtime": 17.4778,
-      "eval_samples_per_second": 2.861,
-      "eval_steps_per_second": 0.744,
+      "eval_runtime": 17.5192,
+      "eval_samples_per_second": 2.854,
+      "eval_steps_per_second": 0.742,
       "step": 10
     },
     {
-      "epoch": 0.09615384615384616,
-      "grad_norm": 7.8091559410095215,
-      "learning_rate": 0.00018805970149253734,
-      "loss": 2.5529,
-      "mean_token_accuracy": 0.9347499996423722,
+      "epoch": 0.5228758169934641,
+      "grad_norm": 57.1182746887207,
+      "learning_rate": 1.8624338624338625e-05,
+      "loss": 17.5658,
+      "mean_token_accuracy": 0.5591249987483025,
       "num_tokens": 98240.0,
       "step": 20
     },
     {
-      "epoch": 0.09615384615384616,
-      "eval_loss": 0.14167079329490662,
-      "eval_mean_token_accuracy": 0.9773077139487634,
+      "epoch": 0.5228758169934641,
+      "eval_loss": 3.730682134628296,
+      "eval_mean_token_accuracy": 0.5742307580434359,
       "eval_num_tokens": 98240.0,
-      "eval_runtime": 17.4221,
-      "eval_samples_per_second": 2.87,
-      "eval_steps_per_second": 0.746,
+      "eval_runtime": 17.4511,
+      "eval_samples_per_second": 2.865,
+      "eval_steps_per_second": 0.745,
       "step": 20
     },
     {
-      "epoch": 0.14423076923076922,
-      "grad_norm": 3.069196939468384,
-      "learning_rate": 0.00017810945273631843,
-      "loss": 0.3788,
-      "mean_token_accuracy": 0.9778750196099282,
+      "epoch": 0.7843137254901961,
+      "grad_norm": 58.24352264404297,
+      "learning_rate": 1.7566137566137566e-05,
+      "loss": 13.113,
+      "mean_token_accuracy": 0.6388749912381172,
       "num_tokens": 147360.0,
       "step": 30
     },
     {
-      "epoch": 0.14423076923076922,
-      "eval_loss": 0.03845745697617531,
-      "eval_mean_token_accuracy": 0.9788461740200336,
+      "epoch": 0.7843137254901961,
+      "eval_loss": 2.7529046535491943,
+      "eval_mean_token_accuracy": 0.7115384431985708,
       "eval_num_tokens": 147360.0,
-      "eval_runtime": 17.5754,
-      "eval_samples_per_second": 2.845,
-      "eval_steps_per_second": 0.74,
+      "eval_runtime": 17.6786,
+      "eval_samples_per_second": 2.828,
+      "eval_steps_per_second": 0.735,
       "step": 30
     },
     {
-      "epoch": 0.19230769230769232,
-      "grad_norm": 6.14934778213501,
-      "learning_rate": 0.0001681592039800995,
-      "loss": 0.1621,
-      "mean_token_accuracy": 0.9760000124573708,
-      "num_tokens": 196480.0,
+      "epoch": 1.026143790849673,
+      "grad_norm": 30.73020362854004,
+      "learning_rate": 1.6507936507936507e-05,
+      "loss": 8.7369,
+      "mean_token_accuracy": 0.7555405417004147,
+      "num_tokens": 192796.0,
       "step": 40
     },
     {
-      "epoch": 0.19230769230769232,
-      "eval_loss": 0.03434291481971741,
-      "eval_mean_token_accuracy": 0.9788461740200336,
-      "eval_num_tokens": 196480.0,
-      "eval_runtime": 17.4843,
-      "eval_samples_per_second": 2.86,
-      "eval_steps_per_second": 0.744,
+      "epoch": 1.026143790849673,
+      "eval_loss": 1.836286187171936,
+      "eval_mean_token_accuracy": 0.8476923199800345,
+      "eval_num_tokens": 192796.0,
+      "eval_runtime": 17.4538,
+      "eval_samples_per_second": 2.865,
+      "eval_steps_per_second": 0.745,
       "step": 40
     },
     {
-      "epoch": 0.2403846153846154,
-      "grad_norm": 7.388150691986084,
-      "learning_rate": 0.00015820895522388059,
-      "loss": 0.1563,
-      "mean_token_accuracy": 0.9780000209808349,
-      "num_tokens": 245600.0,
+      "epoch": 1.287581699346405,
+      "grad_norm": 21.440105438232422,
+      "learning_rate": 1.544973544973545e-05,
+      "loss": 6.2989,
+      "mean_token_accuracy": 0.8789999932050705,
+      "num_tokens": 241916.0,
       "step": 50
     },
     {
-      "epoch": 0.2403846153846154,
-      "eval_loss": 0.05149823799729347,
-      "eval_mean_token_accuracy": 0.9788461740200336,
-      "eval_num_tokens": 245600.0,
-      "eval_runtime": 17.5208,
-      "eval_samples_per_second": 2.854,
-      "eval_steps_per_second": 0.742,
+      "epoch": 1.287581699346405,
+      "eval_loss": 1.3245856761932373,
+      "eval_mean_token_accuracy": 0.8942307508908786,
+      "eval_num_tokens": 241916.0,
+      "eval_runtime": 17.6384,
+      "eval_samples_per_second": 2.835,
+      "eval_steps_per_second": 0.737,
       "step": 50
     },
     {
-      "epoch": 0.28846153846153844,
-      "grad_norm": 5.478137969970703,
-      "learning_rate": 0.00014825870646766168,
-      "loss": 0.177,
-      "mean_token_accuracy": 0.9782500207424164,
-      "num_tokens": 294720.0,
+      "epoch": 1.5490196078431373,
+      "grad_norm": 10.908184051513672,
+      "learning_rate": 1.4391534391534392e-05,
+      "loss": 4.5427,
+      "mean_token_accuracy": 0.9057500049471855,
+      "num_tokens": 291036.0,
       "step": 60
     },
     {
-      "epoch": 0.28846153846153844,
-      "eval_loss": 0.03306853771209717,
-      "eval_mean_token_accuracy": 0.9788461740200336,
-      "eval_num_tokens": 294720.0,
-      "eval_runtime": 17.4723,
+      "epoch": 1.5490196078431373,
+      "eval_loss": 0.9362902045249939,
+      "eval_mean_token_accuracy": 0.9126923221808213,
+      "eval_num_tokens": 291036.0,
+      "eval_runtime": 17.472,
       "eval_samples_per_second": 2.862,
       "eval_steps_per_second": 0.744,
       "step": 60
     },
     {
-      "epoch": 0.33653846153846156,
-      "grad_norm": 4.636706352233887,
-      "learning_rate": 0.0001383084577114428,
-      "loss": 0.1614,
-      "mean_token_accuracy": 0.9776250213384629,
-      "num_tokens": 343840.0,
+      "epoch": 1.8104575163398693,
+      "grad_norm": 11.274917602539062,
+      "learning_rate": 1.3333333333333333e-05,
+      "loss": 3.4538,
+      "mean_token_accuracy": 0.9126250177621842,
+      "num_tokens": 340156.0,
       "step": 70
     },
     {
-      "epoch": 0.33653846153846156,
-      "eval_loss": 0.041035111993551254,
-      "eval_mean_token_accuracy": 0.9788461740200336,
-      "eval_num_tokens": 343840.0,
-      "eval_runtime": 17.567,
-      "eval_samples_per_second": 2.846,
-      "eval_steps_per_second": 0.74,
+      "epoch": 1.8104575163398693,
+      "eval_loss": 0.7567419409751892,
+      "eval_mean_token_accuracy": 0.9103846183189979,
+      "eval_num_tokens": 340156.0,
+      "eval_runtime": 17.5544,
+      "eval_samples_per_second": 2.848,
+      "eval_steps_per_second": 0.741,
       "step": 70
     },
     {
-      "epoch": 0.38461538461538464,
-      "grad_norm": 1.8013979196548462,
-      "learning_rate": 0.00012835820895522389,
-      "loss": 0.1449,
-      "mean_token_accuracy": 0.9785000205039978,
-      "num_tokens": 392960.0,
+      "epoch": 2.052287581699346,
+      "grad_norm": 13.183298110961914,
+      "learning_rate": 1.2275132275132276e-05,
+      "loss": 2.5315,
+      "mean_token_accuracy": 0.9293243321212562,
+      "num_tokens": 385592.0,
       "step": 80
     },
     {
-      "epoch": 0.38461538461538464,
-      "eval_loss": 0.033909473568201065,
-      "eval_mean_token_accuracy": 0.9788461740200336,
-      "eval_num_tokens": 392960.0,
-      "eval_runtime": 17.4599,
-      "eval_samples_per_second": 2.864,
-      "eval_steps_per_second": 0.745,
+      "epoch": 2.052287581699346,
+      "eval_loss": 0.6147329807281494,
+      "eval_mean_token_accuracy": 0.9292307771169223,
+      "eval_num_tokens": 385592.0,
+      "eval_runtime": 17.582,
+      "eval_samples_per_second": 2.844,
+      "eval_steps_per_second": 0.739,
       "step": 80
     },
     {
-      "epoch": 0.4326923076923077,
-      "grad_norm": 0.7925957441329956,
-      "learning_rate": 0.00011840796019900498,
-      "loss": 0.1343,
-      "mean_token_accuracy": 0.979250019788742,
-      "num_tokens": 442080.0,
+      "epoch": 2.313725490196078,
+      "grad_norm": 12.541313171386719,
+      "learning_rate": 1.1216931216931217e-05,
+      "loss": 2.2552,
+      "mean_token_accuracy": 0.9353750020265579,
+      "num_tokens": 434712.0,
       "step": 90
     },
     {
-      "epoch": 0.4326923076923077,
-      "eval_loss": 0.03350621089339256,
-      "eval_mean_token_accuracy": 0.9788461740200336,
-      "eval_num_tokens": 442080.0,
-      "eval_runtime": 17.5312,
-      "eval_samples_per_second": 2.852,
-      "eval_steps_per_second": 0.742,
+      "epoch": 2.313725490196078,
+      "eval_loss": 0.5065863132476807,
+      "eval_mean_token_accuracy": 0.9465384529187129,
+      "eval_num_tokens": 434712.0,
+      "eval_runtime": 17.6818,
+      "eval_samples_per_second": 2.828,
+      "eval_steps_per_second": 0.735,
       "step": 90
     },
     {
-      "epoch": 0.4807692307692308,
-      "grad_norm": 3.3666160106658936,
-      "learning_rate": 0.00010845771144278607,
-      "loss": 0.1484,
-      "mean_token_accuracy": 0.9778750211000442,
-      "num_tokens": 491200.0,
+      "epoch": 2.57516339869281,
+      "grad_norm": 7.007925033569336,
+      "learning_rate": 1.015873015873016e-05,
+      "loss": 1.9547,
+      "mean_token_accuracy": 0.9474999904632568,
+      "num_tokens": 483832.0,
       "step": 100
     },
     {
-      "epoch": 0.4807692307692308,
-      "eval_loss": 0.03422728180885315,
-      "eval_mean_token_accuracy": 0.9788461740200336,
-      "eval_num_tokens": 491200.0,
-      "eval_runtime": 17.4416,
-      "eval_samples_per_second": 2.867,
-      "eval_steps_per_second": 0.745,
+      "epoch": 2.57516339869281,
+      "eval_loss": 0.4519544243812561,
+      "eval_mean_token_accuracy": 0.9511538331325238,
+      "eval_num_tokens": 483832.0,
+      "eval_runtime": 17.5399,
+      "eval_samples_per_second": 2.851,
+      "eval_steps_per_second": 0.741,
       "step": 100
     }
   ],
   "logging_steps": 10,
-  "max_steps": 208,
+  "max_steps": 195,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 1,
+  "num_train_epochs": 5,
   "save_steps": 20,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -217,7 +217,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1.27727482781184e+16,
+  "total_flos": 1.2581157053946624e+16,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null

checkpoint-100/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:09a9baff00283695ce70ca75d7d996c90cc5734afa7d9ed3f9a6113fde47b709
+oid sha256:7fa3299c8cd6826c610bec43e932e8377d63e21789add24e4c875f4d75cb544d
 size 5880