Training in progress, step 400, checkpoint

checkpoint-400/adapter_config.json

@@ -28,16 +28,16 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "fc1",
-    "fc2",
     "o_proj",
+    "fc1",
     "gate_proj",
     "up_proj",
-    "k_proj",
+    "q_proj",
+    "v_proj",
     "down_proj",
+    "k_proj",
     "out_proj",
-    "q_proj",
-    "v_proj"
+    "fc2"
   ],
   "task_type": "CAUSAL_LM",
   "trainable_token_indices": null,

checkpoint-400/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bc79ea31c9f05159694a8268df9027be66e17bbfcb7f85c120751790564a14ef
+oid sha256:36c2d2df5bbc160d2377615ced3af4094c4fc562df7296192de4abd2153f1941
 size 2839126480

checkpoint-400/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:16cb7d77761b46bcbfc6a8b625ffd9ff283ea167c846c6b04f27c95c3674056c
+oid sha256:c04ac2cb09d32c0fff71c15b417d3b7fdefdcaa2f4380033bc13fd468fa77130
 size 5678690152

checkpoint-400/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0ed6d8d369760e08757480bee1347f5f9a123bd537895c933990fed168f6608a
+oid sha256:2123d2cdefe4b1df1b665235b0196d13a18a0218fcb3a07b661248d939b3ceaf
 size 1064

checkpoint-400/trainer_state.json

@@ -11,382 +11,382 @@
   "log_history": [
     {
       "epoch": 0.04162330905306972,
-      "grad_norm": 20.466537475585938,
-      "learning_rate": 4.957081545064378e-05,
-      "loss": 8.8593,
-      "mean_token_accuracy": 0.6828776024281978,
+      "grad_norm": 2.6779890060424805,
+      "learning_rate": 0.00019828326180257511,
+      "loss": 5.368,
+      "mean_token_accuracy": 0.8072595901787281,
       "num_tokens": 161548.0,
       "step": 20
     },
     {
       "epoch": 0.04162330905306972,
-      "eval_loss": 1.1315333843231201,
-      "eval_mean_token_accuracy": 0.8237503719329834,
+      "eval_loss": 0.12305498868227005,
+      "eval_mean_token_accuracy": 0.9847841203212738,
       "eval_num_tokens": 161548.0,
-      "eval_runtime": 34.7122,
-      "eval_samples_per_second": 2.881,
-      "eval_steps_per_second": 0.72,
+      "eval_runtime": 68.9772,
+      "eval_samples_per_second": 2.9,
+      "eval_steps_per_second": 0.725,
       "step": 20
     },
     {
       "epoch": 0.08324661810613944,
-      "grad_norm": 2.744436025619507,
-      "learning_rate": 4.742489270386266e-05,
-      "loss": 1.8553,
-      "mean_token_accuracy": 0.9392111219465733,
+      "grad_norm": 0.5366652011871338,
+      "learning_rate": 0.00018969957081545064,
+      "loss": 0.1516,
+      "mean_token_accuracy": 0.9917096085846424,
       "num_tokens": 323168.0,
       "step": 40
     },
     {
       "epoch": 0.08324661810613944,
-      "eval_loss": 0.16864576935768127,
-      "eval_mean_token_accuracy": 0.9810383248329163,
+      "eval_loss": 0.013321125879883766,
+      "eval_mean_token_accuracy": 0.9938336944580078,
       "eval_num_tokens": 323168.0,
-      "eval_runtime": 34.3084,
-      "eval_samples_per_second": 2.915,
-      "eval_steps_per_second": 0.729,
+      "eval_runtime": 68.8811,
+      "eval_samples_per_second": 2.904,
+      "eval_steps_per_second": 0.726,
       "step": 40
     },
     {
       "epoch": 0.12486992715920915,
-      "grad_norm": 1.6022675037384033,
-      "learning_rate": 4.527896995708155e-05,
-      "loss": 0.3903,
-      "mean_token_accuracy": 0.9888123281300067,
+      "grad_norm": 0.4724605977535248,
+      "learning_rate": 0.0001811158798283262,
+      "loss": 0.0462,
+      "mean_token_accuracy": 0.9939016968011856,
       "num_tokens": 484780.0,
       "step": 60
     },
     {
       "epoch": 0.12486992715920915,
-      "eval_loss": 0.03530249744653702,
-      "eval_mean_token_accuracy": 0.9926152086257934,
+      "eval_loss": 0.010577572509646416,
+      "eval_mean_token_accuracy": 0.99473925948143,
       "eval_num_tokens": 484780.0,
-      "eval_runtime": 34.7758,
-      "eval_samples_per_second": 2.876,
-      "eval_steps_per_second": 0.719,
+      "eval_runtime": 68.7899,
+      "eval_samples_per_second": 2.907,
+      "eval_steps_per_second": 0.727,
       "step": 60
     },
     {
       "epoch": 0.16649323621227888,
-      "grad_norm": 1.2044650316238403,
-      "learning_rate": 4.313304721030043e-05,
-      "loss": 0.0664,
-      "mean_token_accuracy": 0.9934882044792175,
+      "grad_norm": 0.33190304040908813,
+      "learning_rate": 0.00017253218884120172,
+      "loss": 0.0388,
+      "mean_token_accuracy": 0.9948750860989094,
       "num_tokens": 646431.0,
       "step": 80
     },
     {
       "epoch": 0.16649323621227888,
-      "eval_loss": 0.012011010199785233,
-      "eval_mean_token_accuracy": 0.9936227130889893,
+      "eval_loss": 0.00940256379544735,
+      "eval_mean_token_accuracy": 0.9951227140426636,
       "eval_num_tokens": 646431.0,
-      "eval_runtime": 34.3329,
-      "eval_samples_per_second": 2.913,
-      "eval_steps_per_second": 0.728,
+      "eval_runtime": 69.1687,
+      "eval_samples_per_second": 2.891,
+      "eval_steps_per_second": 0.723,
       "step": 80
     },
     {
       "epoch": 0.2081165452653486,
-      "grad_norm": 0.4138229191303253,
-      "learning_rate": 4.098712446351932e-05,
-      "loss": 0.0513,
-      "mean_token_accuracy": 0.994041533768177,
+      "grad_norm": 0.20264093577861786,
+      "learning_rate": 0.00016394849785407727,
+      "loss": 0.0367,
+      "mean_token_accuracy": 0.9950500458478928,
       "num_tokens": 808054.0,
       "step": 100
     },
     {
       "epoch": 0.2081165452653486,
-      "eval_loss": 0.011702906340360641,
-      "eval_mean_token_accuracy": 0.9942703366279602,
+      "eval_loss": 0.008246215991675854,
+      "eval_mean_token_accuracy": 0.9951834440231323,
       "eval_num_tokens": 808054.0,
-      "eval_runtime": 34.3225,
-      "eval_samples_per_second": 2.914,
+      "eval_runtime": 68.7059,
+      "eval_samples_per_second": 2.911,
       "eval_steps_per_second": 0.728,
       "step": 100
     },
     {
       "epoch": 0.2497398543184183,
-      "grad_norm": 0.8489187955856323,
-      "learning_rate": 3.88412017167382e-05,
-      "loss": 0.0416,
-      "mean_token_accuracy": 0.9943524189293385,
+      "grad_norm": 0.30675262212753296,
+      "learning_rate": 0.0001553648068669528,
+      "loss": 0.0341,
+      "mean_token_accuracy": 0.994870014488697,
       "num_tokens": 969623.0,
       "step": 120
     },
     {
       "epoch": 0.2497398543184183,
-      "eval_loss": 0.010666043497622013,
-      "eval_mean_token_accuracy": 0.9943506979942321,
+      "eval_loss": 0.00862042885273695,
+      "eval_mean_token_accuracy": 0.994941633939743,
       "eval_num_tokens": 969623.0,
-      "eval_runtime": 34.3761,
-      "eval_samples_per_second": 2.909,
+      "eval_runtime": 68.8073,
+      "eval_samples_per_second": 2.907,
       "eval_steps_per_second": 0.727,
       "step": 120
     },
     {
       "epoch": 0.29136316337148804,
-      "grad_norm": 0.49626249074935913,
-      "learning_rate": 3.669527896995708e-05,
-      "loss": 0.0384,
-      "mean_token_accuracy": 0.9943184182047844,
+      "grad_norm": 0.1850423365831375,
+      "learning_rate": 0.00014678111587982832,
+      "loss": 0.034,
+      "mean_token_accuracy": 0.9949115067720413,
       "num_tokens": 1131222.0,
       "step": 140
     },
     {
       "epoch": 0.29136316337148804,
-      "eval_loss": 0.009807135909795761,
-      "eval_mean_token_accuracy": 0.9946736145019531,
+      "eval_loss": 0.007929541170597076,
+      "eval_mean_token_accuracy": 0.9951228404045105,
       "eval_num_tokens": 1131222.0,
-      "eval_runtime": 34.34,
-      "eval_samples_per_second": 2.912,
+      "eval_runtime": 68.7004,
+      "eval_samples_per_second": 2.911,
       "eval_steps_per_second": 0.728,
       "step": 140
     },
     {
       "epoch": 0.33298647242455776,
-      "grad_norm": 0.5716305375099182,
-      "learning_rate": 3.454935622317597e-05,
-      "loss": 0.0384,
-      "mean_token_accuracy": 0.9946457795798779,
+      "grad_norm": 0.3047815263271332,
+      "learning_rate": 0.00013819742489270387,
+      "loss": 0.0328,
+      "mean_token_accuracy": 0.9950883395969867,
       "num_tokens": 1292839.0,
       "step": 160
     },
     {
       "epoch": 0.33298647242455776,
-      "eval_loss": 0.009297176264226437,
-      "eval_mean_token_accuracy": 0.9946328043937683,
+      "eval_loss": 0.007993862964212894,
+      "eval_mean_token_accuracy": 0.9951430022716522,
       "eval_num_tokens": 1292839.0,
-      "eval_runtime": 34.557,
-      "eval_samples_per_second": 2.894,
-      "eval_steps_per_second": 0.723,
+      "eval_runtime": 68.1668,
+      "eval_samples_per_second": 2.934,
+      "eval_steps_per_second": 0.733,
       "step": 160
     },
     {
       "epoch": 0.37460978147762747,
-      "grad_norm": 0.4264802932739258,
-      "learning_rate": 3.240343347639485e-05,
-      "loss": 0.0373,
-      "mean_token_accuracy": 0.9946053452789784,
+      "grad_norm": 0.13929370045661926,
+      "learning_rate": 0.0001296137339055794,
+      "loss": 0.0331,
+      "mean_token_accuracy": 0.9951471641659737,
       "num_tokens": 1454417.0,
       "step": 180
     },
     {
       "epoch": 0.37460978147762747,
-      "eval_loss": 0.009250417351722717,
-      "eval_mean_token_accuracy": 0.9949553918838501,
+      "eval_loss": 0.007801079656928778,
+      "eval_mean_token_accuracy": 0.9954658925533295,
       "eval_num_tokens": 1454417.0,
-      "eval_runtime": 34.3425,
-      "eval_samples_per_second": 2.912,
-      "eval_steps_per_second": 0.728,
+      "eval_runtime": 68.0889,
+      "eval_samples_per_second": 2.937,
+      "eval_steps_per_second": 0.734,
       "step": 180
     },
     {
       "epoch": 0.4162330905306972,
-      "grad_norm": 0.3315845727920532,
-      "learning_rate": 3.0257510729613737e-05,
-      "loss": 0.0326,
-      "mean_token_accuracy": 0.9949214711785317,
+      "grad_norm": 0.29847878217697144,
+      "learning_rate": 0.00012103004291845495,
+      "loss": 0.0308,
+      "mean_token_accuracy": 0.9953480623662472,
       "num_tokens": 1615982.0,
       "step": 200
     },
     {
       "epoch": 0.4162330905306972,
-      "eval_loss": 0.00862209778279066,
-      "eval_mean_token_accuracy": 0.994794466495514,
+      "eval_loss": 0.007313677109777927,
+      "eval_mean_token_accuracy": 0.9954455089569092,
       "eval_num_tokens": 1615982.0,
-      "eval_runtime": 34.317,
-      "eval_samples_per_second": 2.914,
-      "eval_steps_per_second": 0.729,
+      "eval_runtime": 68.4814,
+      "eval_samples_per_second": 2.921,
+      "eval_steps_per_second": 0.73,
       "step": 200
     },
     {
       "epoch": 0.4578563995837669,
-      "grad_norm": 0.3415542542934418,
-      "learning_rate": 2.811158798283262e-05,
-      "loss": 0.0328,
-      "mean_token_accuracy": 0.9948225237429142,
+      "grad_norm": 0.14924530684947968,
+      "learning_rate": 0.00011244635193133049,
+      "loss": 0.0298,
+      "mean_token_accuracy": 0.9952255949378014,
       "num_tokens": 1777586.0,
       "step": 220
     },
     {
       "epoch": 0.4578563995837669,
-      "eval_loss": 0.008552273735404015,
-      "eval_mean_token_accuracy": 0.9952386736869812,
+      "eval_loss": 0.0077740405686199665,
+      "eval_mean_token_accuracy": 0.9953651452064514,
       "eval_num_tokens": 1777586.0,
-      "eval_runtime": 34.357,
-      "eval_samples_per_second": 2.911,
-      "eval_steps_per_second": 0.728,
+      "eval_runtime": 68.0902,
+      "eval_samples_per_second": 2.937,
+      "eval_steps_per_second": 0.734,
       "step": 220
     },
     {
       "epoch": 0.4994797086368366,
-      "grad_norm": 0.5801106691360474,
-      "learning_rate": 2.59656652360515e-05,
-      "loss": 0.0317,
-      "mean_token_accuracy": 0.9949369013309479,
+      "grad_norm": 0.31947973370552063,
+      "learning_rate": 0.000103862660944206,
+      "loss": 0.0299,
+      "mean_token_accuracy": 0.9952647179365158,
       "num_tokens": 1939204.0,
       "step": 240
     },
     {
       "epoch": 0.4994797086368366,
-      "eval_loss": 0.008411003276705742,
-      "eval_mean_token_accuracy": 0.9951176619529725,
+      "eval_loss": 0.007334359921514988,
+      "eval_mean_token_accuracy": 0.9954258131980896,
       "eval_num_tokens": 1939204.0,
-      "eval_runtime": 34.3383,
-      "eval_samples_per_second": 2.912,
-      "eval_steps_per_second": 0.728,
+      "eval_runtime": 68.026,
+      "eval_samples_per_second": 2.94,
+      "eval_steps_per_second": 0.735,
       "step": 240
     },
     {
       "epoch": 0.5411030176899063,
-      "grad_norm": 0.3876211941242218,
-      "learning_rate": 2.3819742489270388e-05,
-      "loss": 0.0322,
-      "mean_token_accuracy": 0.9952259331941604,
+      "grad_norm": 0.15298807621002197,
+      "learning_rate": 9.527896995708155e-05,
+      "loss": 0.0305,
+      "mean_token_accuracy": 0.9951752610504627,
       "num_tokens": 2100821.0,
       "step": 260
     },
     {
       "epoch": 0.5411030176899063,
-      "eval_loss": 0.008060808293521404,
-      "eval_mean_token_accuracy": 0.9952384281158447,
+      "eval_loss": 0.007329077925533056,
+      "eval_mean_token_accuracy": 0.9955062341690063,
       "eval_num_tokens": 2100821.0,
-      "eval_runtime": 34.5381,
-      "eval_samples_per_second": 2.895,
-      "eval_steps_per_second": 0.724,
+      "eval_runtime": 68.3167,
+      "eval_samples_per_second": 2.928,
+      "eval_steps_per_second": 0.732,
       "step": 260
     },
     {
       "epoch": 0.5827263267429761,
-      "grad_norm": 0.3304857015609741,
-      "learning_rate": 2.1673819742489272e-05,
-      "loss": 0.0326,
-      "mean_token_accuracy": 0.9948949187994003,
+      "grad_norm": 0.1927741914987564,
+      "learning_rate": 8.669527896995709e-05,
+      "loss": 0.0301,
+      "mean_token_accuracy": 0.9952726632356643,
       "num_tokens": 2262372.0,
       "step": 280
     },
     {
       "epoch": 0.5827263267429761,
-      "eval_loss": 0.008030685596168041,
-      "eval_mean_token_accuracy": 0.9953594398498535,
+      "eval_loss": 0.007559146732091904,
+      "eval_mean_token_accuracy": 0.9953450250625611,
       "eval_num_tokens": 2262372.0,
-      "eval_runtime": 34.2589,
-      "eval_samples_per_second": 2.919,
-      "eval_steps_per_second": 0.73,
+      "eval_runtime": 68.1437,
+      "eval_samples_per_second": 2.935,
+      "eval_steps_per_second": 0.734,
       "step": 280
     },
     {
       "epoch": 0.6243496357960457,
-      "grad_norm": 0.262820839881897,
-      "learning_rate": 1.9527896995708157e-05,
-      "loss": 0.0295,
-      "mean_token_accuracy": 0.9954247616231442,
+      "grad_norm": 0.15931323170661926,
+      "learning_rate": 7.811158798283263e-05,
+      "loss": 0.028,
+      "mean_token_accuracy": 0.9956002615392208,
       "num_tokens": 2423931.0,
       "step": 300
     },
     {
       "epoch": 0.6243496357960457,
-      "eval_loss": 0.0076558589935302734,
-      "eval_mean_token_accuracy": 0.9952384281158447,
+      "eval_loss": 0.007200200576335192,
+      "eval_mean_token_accuracy": 0.9955260717868805,
       "eval_num_tokens": 2423931.0,
-      "eval_runtime": 34.2994,
-      "eval_samples_per_second": 2.915,
-      "eval_steps_per_second": 0.729,
+      "eval_runtime": 68.2059,
+      "eval_samples_per_second": 2.932,
+      "eval_steps_per_second": 0.733,
       "step": 300
     },
     {
       "epoch": 0.6659729448491155,
-      "grad_norm": 0.4573795199394226,
-      "learning_rate": 1.7381974248927038e-05,
-      "loss": 0.0308,
-      "mean_token_accuracy": 0.9953766994178295,
+      "grad_norm": 0.2115289717912674,
+      "learning_rate": 6.952789699570815e-05,
+      "loss": 0.0274,
+      "mean_token_accuracy": 0.9953135840594769,
       "num_tokens": 2585532.0,
       "step": 320
     },
     {
       "epoch": 0.6659729448491155,
-      "eval_loss": 0.007896814495325089,
-      "eval_mean_token_accuracy": 0.9952785062789917,
+      "eval_loss": 0.00743951927870512,
+      "eval_mean_token_accuracy": 0.9954253661632538,
       "eval_num_tokens": 2585532.0,
-      "eval_runtime": 34.2979,
-      "eval_samples_per_second": 2.916,
-      "eval_steps_per_second": 0.729,
+      "eval_runtime": 67.978,
+      "eval_samples_per_second": 2.942,
+      "eval_steps_per_second": 0.736,
       "step": 320
     },
     {
       "epoch": 0.7075962539021852,
-      "grad_norm": 0.3572976887226105,
-      "learning_rate": 1.5236051502145923e-05,
-      "loss": 0.0311,
-      "mean_token_accuracy": 0.9952577523887157,
+      "grad_norm": 0.19469623267650604,
+      "learning_rate": 6.094420600858369e-05,
+      "loss": 0.0291,
+      "mean_token_accuracy": 0.9953210979700089,
       "num_tokens": 2747068.0,
       "step": 340
     },
     {
       "epoch": 0.7075962539021852,
-      "eval_loss": 0.007795471698045731,
-      "eval_mean_token_accuracy": 0.9952789568901061,
+      "eval_loss": 0.0072942511178553104,
+      "eval_mean_token_accuracy": 0.9953449034690857,
       "eval_num_tokens": 2747068.0,
-      "eval_runtime": 34.3024,
+      "eval_runtime": 68.6035,
       "eval_samples_per_second": 2.915,
       "eval_steps_per_second": 0.729,
       "step": 340
     },
     {
       "epoch": 0.7492195629552549,
-      "grad_norm": 0.25351160764694214,
-      "learning_rate": 1.3090128755364809e-05,
-      "loss": 0.0302,
-      "mean_token_accuracy": 0.9953413404524326,
+      "grad_norm": 0.14993388950824738,
+      "learning_rate": 5.2360515021459236e-05,
+      "loss": 0.029,
+      "mean_token_accuracy": 0.9953904427587986,
       "num_tokens": 2908697.0,
       "step": 360
     },
     {
       "epoch": 0.7492195629552549,
-      "eval_loss": 0.007526129484176636,
-      "eval_mean_token_accuracy": 0.9951572942733765,
+      "eval_loss": 0.0072441427037119865,
+      "eval_mean_token_accuracy": 0.9954857683181763,
       "eval_num_tokens": 2908697.0,
-      "eval_runtime": 34.4802,
-      "eval_samples_per_second": 2.9,
-      "eval_steps_per_second": 0.725,
+      "eval_runtime": 68.0534,
+      "eval_samples_per_second": 2.939,
+      "eval_steps_per_second": 0.735,
       "step": 360
     },
     {
       "epoch": 0.7908428720083247,
-      "grad_norm": 0.45476171374320984,
-      "learning_rate": 1.0944206008583692e-05,
-      "loss": 0.0301,
-      "mean_token_accuracy": 0.9954766884446145,
+      "grad_norm": 0.19807656109333038,
+      "learning_rate": 4.377682403433477e-05,
+      "loss": 0.0289,
+      "mean_token_accuracy": 0.9953253343701363,
       "num_tokens": 3070284.0,
       "step": 380
     },
     {
       "epoch": 0.7908428720083247,
-      "eval_loss": 0.007462680339813232,
-      "eval_mean_token_accuracy": 0.9952787923812866,
+      "eval_loss": 0.00713132182136178,
+      "eval_mean_token_accuracy": 0.9954454469680786,
       "eval_num_tokens": 3070284.0,
-      "eval_runtime": 34.2958,
-      "eval_samples_per_second": 2.916,
-      "eval_steps_per_second": 0.729,
+      "eval_runtime": 68.0742,
+      "eval_samples_per_second": 2.938,
+      "eval_steps_per_second": 0.734,
       "step": 380
     },
     {
       "epoch": 0.8324661810613944,
-      "grad_norm": 0.37078818678855896,
-      "learning_rate": 8.798283261802575e-06,
-      "loss": 0.0298,
-      "mean_token_accuracy": 0.9952507764101028,
+      "grad_norm": 0.124539814889431,
+      "learning_rate": 3.51931330472103e-05,
+      "loss": 0.0282,
+      "mean_token_accuracy": 0.9952500656247139,
       "num_tokens": 3231892.0,
       "step": 400
     },
     {
       "epoch": 0.8324661810613944,
-      "eval_loss": 0.0073690456338226795,
-      "eval_mean_token_accuracy": 0.9953188729286194,
+      "eval_loss": 0.007116459775716066,
+      "eval_mean_token_accuracy": 0.995526316165924,
       "eval_num_tokens": 3231892.0,
-      "eval_runtime": 34.3243,
-      "eval_samples_per_second": 2.913,
-      "eval_steps_per_second": 0.728,
+      "eval_runtime": 68.1093,
+      "eval_samples_per_second": 2.936,
+      "eval_steps_per_second": 0.734,
       "step": 400
     }
   ],
@@ -394,7 +394,7 @@
   "max_steps": 481,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
-  "save_steps": 50,
+  "save_steps": 40,
   "stateful_callbacks": {
     "TrainerControl": {
       "args": {

checkpoint-400/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dd8015c9bbe5c0dbff1905684d15785ccb3cb07ead997cd50301f35dcf48b143
-size 5880
+oid sha256:4a99c92db5718c8e3fa68a50a104ce7f740a033660d2ea251fbb6febbc7e4942
+size 5816