Model save

README.md

@@ -34,11 +34,11 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 0.0001
-- train_batch_size: 4
+- train_batch_size: 16
 - eval_batch_size: 8
 - seed: 42
 - gradient_accumulation_steps: 16
-- total_train_batch_size: 64
+- total_train_batch_size: 256
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
 - lr_scheduler_warmup_steps: 2

adapter_config.json

@@ -16,17 +16,17 @@
   "megatron_core": "megatron.core",
   "modules_to_save": null,
   "peft_type": "LORA",
-  "r": 16,
+  "r": 8,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
+    "down_proj",
+    "up_proj",
     "q_proj",
     "gate_proj",
-    "k_proj",
-    "v_proj",
-    "up_proj",
     "o_proj",
-    "down_proj"
+    "v_proj",
+    "k_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a4795bfd27ceca59a95652b7f3315d7cef985190ec0e31ff2d89a90801c6ccef
-size 1103312056
+oid sha256:5b76d8288879581778f246b3845bced1ecc2a578d3d54dae5edab913ca5d5b73
+size 1080767344

trainer_state.json

@@ -3,247 +3,79 @@
   "best_model_checkpoint": null,
   "epoch": 4.0,
   "eval_steps": 500,
-  "global_step": 8240,
+  "global_step": 2060,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
-    {
-      "epoch": 0.12135922330097088,
-      "grad_norm": 1.0826135873794556,
-      "learning_rate": 9.698956057295461e-05,
-      "loss": 1.259,
-      "step": 250
-    },
-    {
-      "epoch": 0.24271844660194175,
-      "grad_norm": 1.2716737985610962,
-      "learning_rate": 9.395484340859432e-05,
-      "loss": 0.7161,
-      "step": 500
-    },
-    {
-      "epoch": 0.3640776699029126,
-      "grad_norm": 1.1046184301376343,
-      "learning_rate": 9.092012624423404e-05,
-      "loss": 0.5956,
-      "step": 750
-    },
     {
       "epoch": 0.4854368932038835,
-      "grad_norm": 1.0273100137710571,
-      "learning_rate": 8.788540907987377e-05,
-      "loss": 0.5399,
-      "step": 1000
-    },
-    {
-      "epoch": 0.6067961165048543,
-      "grad_norm": 0.8700250387191772,
-      "learning_rate": 8.485069191551348e-05,
-      "loss": 0.5162,
-      "step": 1250
-    },
-    {
-      "epoch": 0.7281553398058253,
-      "grad_norm": 0.8778573274612427,
-      "learning_rate": 8.181597475115321e-05,
-      "loss": 0.4971,
-      "step": 1500
-    },
-    {
-      "epoch": 0.8495145631067961,
-      "grad_norm": 0.8028275370597839,
-      "learning_rate": 7.878125758679291e-05,
-      "loss": 0.4859,
-      "step": 1750
+      "grad_norm": 0.5794751644134521,
+      "learning_rate": 8.794946550048592e-05,
+      "loss": 0.9726,
+      "step": 250
     },
     {
       "epoch": 0.970873786407767,
-      "grad_norm": 0.7014565467834473,
-      "learning_rate": 7.574654042243264e-05,
-      "loss": 0.4794,
-      "step": 2000
-    },
-    {
-      "epoch": 1.0922330097087378,
-      "grad_norm": 0.7191192507743835,
-      "learning_rate": 7.271182325807235e-05,
-      "loss": 0.4731,
-      "step": 2250
-    },
-    {
-      "epoch": 1.2135922330097086,
-      "grad_norm": 0.7179076075553894,
-      "learning_rate": 6.967710609371208e-05,
-      "loss": 0.4647,
-      "step": 2500
-    },
-    {
-      "epoch": 1.3349514563106797,
-      "grad_norm": 0.6208838820457458,
-      "learning_rate": 6.664238892935178e-05,
-      "loss": 0.4659,
-      "step": 2750
+      "grad_norm": 0.8225833773612976,
+      "learning_rate": 7.580174927113704e-05,
+      "loss": 0.6738,
+      "step": 500
     },
     {
       "epoch": 1.4563106796116505,
-      "grad_norm": 0.6834104061126709,
-      "learning_rate": 6.360767176499151e-05,
-      "loss": 0.4601,
-      "step": 3000
-    },
-    {
-      "epoch": 1.5776699029126213,
-      "grad_norm": 0.7568572163581848,
-      "learning_rate": 6.0572954600631224e-05,
-      "loss": 0.4533,
-      "step": 3250
-    },
-    {
-      "epoch": 1.6990291262135924,
-      "grad_norm": 0.7566137909889221,
-      "learning_rate": 5.7538237436270945e-05,
-      "loss": 0.4465,
-      "step": 3500
-    },
-    {
-      "epoch": 1.820388349514563,
-      "grad_norm": 0.7088573575019836,
-      "learning_rate": 5.450352027191066e-05,
-      "loss": 0.439,
-      "step": 3750
+      "grad_norm": 0.9149301648139954,
+      "learning_rate": 6.365403304178815e-05,
+      "loss": 0.5896,
+      "step": 750
     },
     {
       "epoch": 1.941747572815534,
-      "grad_norm": 0.7710273265838623,
-      "learning_rate": 5.146880310755038e-05,
-      "loss": 0.4336,
-      "step": 4000
-    },
-    {
-      "epoch": 2.063106796116505,
-      "grad_norm": 0.8484824299812317,
-      "learning_rate": 4.84340859431901e-05,
-      "loss": 0.4197,
-      "step": 4250
-    },
-    {
-      "epoch": 2.1844660194174756,
-      "grad_norm": 0.877728283405304,
-      "learning_rate": 4.539936877882982e-05,
-      "loss": 0.4089,
-      "step": 4500
-    },
-    {
-      "epoch": 2.3058252427184467,
-      "grad_norm": 0.9526512622833252,
-      "learning_rate": 4.236465161446954e-05,
-      "loss": 0.4015,
-      "step": 4750
+      "grad_norm": 1.0453987121582031,
+      "learning_rate": 5.150631681243926e-05,
+      "loss": 0.5344,
+      "step": 1000
     },
     {
       "epoch": 2.4271844660194173,
-      "grad_norm": 1.0069864988327026,
-      "learning_rate": 3.932993445010925e-05,
-      "loss": 0.3921,
-      "step": 5000
-    },
-    {
-      "epoch": 2.5485436893203883,
-      "grad_norm": 0.9432169198989868,
-      "learning_rate": 3.6295217285748975e-05,
-      "loss": 0.3812,
-      "step": 5250
-    },
-    {
-      "epoch": 2.6699029126213594,
-      "grad_norm": 1.0197330713272095,
-      "learning_rate": 3.326050012138869e-05,
-      "loss": 0.3739,
-      "step": 5500
-    },
-    {
-      "epoch": 2.79126213592233,
-      "grad_norm": 1.1332706212997437,
-      "learning_rate": 3.022578295702841e-05,
-      "loss": 0.367,
-      "step": 5750
+      "grad_norm": 0.9058604836463928,
+      "learning_rate": 3.9358600583090386e-05,
+      "loss": 0.4996,
+      "step": 1250
     },
     {
       "epoch": 2.912621359223301,
-      "grad_norm": 1.0419621467590332,
-      "learning_rate": 2.7191065792668125e-05,
-      "loss": 0.3579,
-      "step": 6000
-    },
-    {
-      "epoch": 3.033980582524272,
-      "grad_norm": 1.1409462690353394,
-      "learning_rate": 2.4156348628307843e-05,
-      "loss": 0.3471,
-      "step": 6250
-    },
-    {
-      "epoch": 3.1553398058252426,
-      "grad_norm": 1.171637773513794,
-      "learning_rate": 2.112163146394756e-05,
-      "loss": 0.3316,
-      "step": 6500
-    },
-    {
-      "epoch": 3.2766990291262137,
-      "grad_norm": 1.1024283170700073,
-      "learning_rate": 1.808691429958728e-05,
-      "loss": 0.3269,
-      "step": 6750
+      "grad_norm": 1.047943353652954,
+      "learning_rate": 2.72108843537415e-05,
+      "loss": 0.4784,
+      "step": 1500
     },
     {
       "epoch": 3.3980582524271843,
-      "grad_norm": 1.1913949251174927,
-      "learning_rate": 1.5052197135226997e-05,
-      "loss": 0.3233,
-      "step": 7000
-    },
-    {
-      "epoch": 3.5194174757281553,
-      "grad_norm": 1.1978884935379028,
-      "learning_rate": 1.2017479970866715e-05,
-      "loss": 0.3199,
-      "step": 7250
-    },
-    {
-      "epoch": 3.6407766990291264,
-      "grad_norm": 1.165363073348999,
-      "learning_rate": 8.982762806506435e-06,
-      "loss": 0.3173,
-      "step": 7500
-    },
-    {
-      "epoch": 3.762135922330097,
-      "grad_norm": 1.1788644790649414,
-      "learning_rate": 5.948045642146152e-06,
-      "loss": 0.3127,
-      "step": 7750
+      "grad_norm": 0.7062876224517822,
+      "learning_rate": 1.5063168124392615e-05,
+      "loss": 0.4613,
+      "step": 1750
     },
     {
       "epoch": 3.883495145631068,
-      "grad_norm": 1.089274525642395,
-      "learning_rate": 2.9133284777858704e-06,
-      "loss": 0.3128,
-      "step": 8000
+      "grad_norm": 0.6118430495262146,
+      "learning_rate": 2.915451895043732e-06,
+      "loss": 0.454,
+      "step": 2000
     },
     {
       "epoch": 4.0,
-      "step": 8240,
-      "total_flos": 3.837880094282465e+17,
-      "train_loss": 0.4465005939446607,
-      "train_runtime": 12679.4227,
-      "train_samples_per_second": 41.591,
-      "train_steps_per_second": 0.65
+      "step": 2060,
+      "total_flos": 4.996840775351132e+17,
+      "train_loss": 0.5791272876332108,
+      "train_runtime": 7755.537,
+      "train_samples_per_second": 67.996,
+      "train_steps_per_second": 0.266
     }
   ],
   "logging_steps": 250,
-  "max_steps": 8240,
+  "max_steps": 2060,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 4,
   "save_steps": 500,
@@ -259,8 +91,8 @@
       "attributes": {}
     }
   },
-  "total_flos": 3.837880094282465e+17,
-  "train_batch_size": 4,
+  "total_flos": 4.996840775351132e+17,
+  "train_batch_size": 16,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5fa8edf71a0a6eb97ad6ceed7ec5d2805679572b31ed3d20dea35d2b69c53fa1
+oid sha256:a1c939db3287c1e42257893106e71bc5032cc86e31cb9e70d947062d5cf0fdd9
 size 5240