Model save

README.md

@@ -27,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/wxnfifth/huggingface/runs/pu6294f1) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/wxnfifth/huggingface/runs/j55yx0w5) 
 
 
 This model was trained with SFT.
@@ -35,7 +35,7 @@ This model was trained with SFT.
 ### Framework versions
 
 - TRL: 0.15.0.dev0
-- Transformers: 4.49.0.dev0
+- Transformers: 4.48.2
 - Pytorch: 2.5.1
 - Datasets: 3.2.0
 - Tokenizers: 0.21.0

all_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.997779422649889,
     "total_flos": 76745898196992.0,
-    "train_loss": 0.7814551896086787,
-    "train_runtime": 4577.2961,
+    "train_loss": 0.7870922902217604,
+    "train_runtime": 4743.2538,
     "train_samples": 16610,
-    "train_samples_per_second": 4.721,
-    "train_steps_per_second": 0.074
+    "train_samples_per_second": 4.556,
+    "train_steps_per_second": 0.071
 }

config.json

@@ -22,7 +22,7 @@
   "sliding_window": null,
   "tie_word_embeddings": true,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.49.0.dev0",
+  "transformers_version": "4.48.2",
   "use_cache": false,
   "use_sliding_window": false,
   "vocab_size": 151936

generation_config.json

@@ -10,5 +10,5 @@
   "temperature": 0.7,
   "top_k": 20,
   "top_p": 0.8,
-  "transformers_version": "4.49.0.dev0"
+  "transformers_version": "4.48.2"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a72b2d5bb187809d5bb9e78e96673a081df0491bc4402af7547768018f99387f
+oid sha256:9c30079c59ee74a761bb370b7482619c19b48ace8a8c63e17b6b0cadc3f47833
 size 3087467144

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 0.997779422649889,
     "total_flos": 76745898196992.0,
-    "train_loss": 0.7814551896086787,
-    "train_runtime": 4577.2961,
+    "train_loss": 0.7870922902217604,
+    "train_runtime": 4743.2538,
     "train_samples": 16610,
-    "train_samples_per_second": 4.721,
-    "train_steps_per_second": 0.074
+    "train_samples_per_second": 4.556,
+    "train_steps_per_second": 0.071
 }

trainer_state.json

@@ -10,505 +10,505 @@
   "log_history": [
     {
       "epoch": 0.014803849000740192,
-      "grad_norm": 2.6030564308166504,
+      "grad_norm": 0.6502017974853516,
       "learning_rate": 2.9411764705882355e-06,
       "loss": 1.09,
       "step": 5
     },
     {
       "epoch": 0.029607698001480384,
-      "grad_norm": 1.507163405418396,
+      "grad_norm": 0.3823283016681671,
       "learning_rate": 5.882352941176471e-06,
-      "loss": 1.0787,
+      "loss": 1.0792,
       "step": 10
     },
     {
       "epoch": 0.04441154700222058,
-      "grad_norm": 1.2673176527023315,
+      "grad_norm": 0.3939237594604492,
       "learning_rate": 8.823529411764707e-06,
-      "loss": 1.0147,
+      "loss": 1.0223,
       "step": 15
     },
     {
       "epoch": 0.05921539600296077,
-      "grad_norm": 0.8998919725418091,
+      "grad_norm": 0.2809950113296509,
       "learning_rate": 1.1764705882352942e-05,
-      "loss": 0.9334,
+      "loss": 0.9451,
       "step": 20
     },
     {
       "epoch": 0.07401924500370097,
-      "grad_norm": 0.7936584949493408,
+      "grad_norm": 0.22726929187774658,
       "learning_rate": 1.4705882352941179e-05,
-      "loss": 0.9,
+      "loss": 0.9125,
       "step": 25
     },
     {
       "epoch": 0.08882309400444116,
-      "grad_norm": 0.6800016164779663,
+      "grad_norm": 0.17815199494361877,
       "learning_rate": 1.7647058823529414e-05,
-      "loss": 0.8802,
+      "loss": 0.893,
       "step": 30
     },
     {
       "epoch": 0.10362694300518134,
-      "grad_norm": 0.6241341829299927,
+      "grad_norm": 0.1736987829208374,
       "learning_rate": 1.9999462497359468e-05,
-      "loss": 0.8536,
+      "loss": 0.8651,
       "step": 35
     },
     {
       "epoch": 0.11843079200592153,
-      "grad_norm": 0.542241096496582,
+      "grad_norm": 0.14923641085624695,
       "learning_rate": 1.9980655971335944e-05,
-      "loss": 0.8348,
+      "loss": 0.8452,
       "step": 40
     },
     {
       "epoch": 0.13323464100666174,
-      "grad_norm": 0.511304497718811,
+      "grad_norm": 0.12440051883459091,
       "learning_rate": 1.993503206718859e-05,
-      "loss": 0.8136,
+      "loss": 0.8228,
       "step": 45
     },
     {
       "epoch": 0.14803849000740193,
-      "grad_norm": 0.6061602234840393,
+      "grad_norm": 0.14250266551971436,
       "learning_rate": 1.986271337340182e-05,
-      "loss": 0.8197,
+      "loss": 0.8278,
       "step": 50
     },
     {
       "epoch": 0.16284233900814213,
-      "grad_norm": 0.5142273306846619,
+      "grad_norm": 0.12134958058595657,
       "learning_rate": 1.976389420563607e-05,
-      "loss": 0.8027,
+      "loss": 0.8105,
       "step": 55
     },
     {
       "epoch": 0.17764618800888232,
-      "grad_norm": 0.494143009185791,
+      "grad_norm": 0.1252334713935852,
       "learning_rate": 1.9638840084614182e-05,
-      "loss": 0.7893,
+      "loss": 0.7963,
       "step": 60
     },
     {
       "epoch": 0.19245003700962252,
-      "grad_norm": 0.5752012729644775,
+      "grad_norm": 0.12292112410068512,
       "learning_rate": 1.9487887022684336e-05,
-      "loss": 0.7993,
+      "loss": 0.8063,
       "step": 65
     },
     {
       "epoch": 0.20725388601036268,
-      "grad_norm": 0.5229590535163879,
+      "grad_norm": 0.13962922990322113,
       "learning_rate": 1.9311440620976597e-05,
-      "loss": 0.7927,
+      "loss": 0.799,
       "step": 70
     },
     {
       "epoch": 0.22205773501110287,
-      "grad_norm": 0.5390455722808838,
+      "grad_norm": 0.1244792640209198,
       "learning_rate": 1.9109974979578852e-05,
-      "loss": 0.7836,
+      "loss": 0.7898,
       "step": 75
     },
     {
       "epoch": 0.23686158401184307,
-      "grad_norm": 0.5431194305419922,
+      "grad_norm": 0.12688860297203064,
       "learning_rate": 1.8884031423660492e-05,
-      "loss": 0.8122,
+      "loss": 0.8185,
       "step": 80
     },
     {
       "epoch": 0.25166543301258326,
-      "grad_norm": 0.5343985557556152,
+      "grad_norm": 0.12278366833925247,
       "learning_rate": 1.8634217048966638e-05,
-      "loss": 0.795,
+      "loss": 0.801,
       "step": 85
     },
     {
       "epoch": 0.2664692820133235,
-      "grad_norm": 0.5408906936645508,
+      "grad_norm": 0.11767291277647018,
       "learning_rate": 1.836120309059107e-05,
-      "loss": 0.7778,
+      "loss": 0.7838,
       "step": 90
     },
     {
       "epoch": 0.28127313101406365,
-      "grad_norm": 0.5261913537979126,
+      "grad_norm": 0.12599965929985046,
       "learning_rate": 1.8065723119410885e-05,
-      "loss": 0.7756,
+      "loss": 0.7808,
       "step": 95
     },
     {
       "epoch": 0.29607698001480387,
-      "grad_norm": 0.5498437881469727,
+      "grad_norm": 0.13216127455234528,
       "learning_rate": 1.77485710710289e-05,
-      "loss": 0.7824,
+      "loss": 0.7879,
       "step": 100
     },
     {
       "epoch": 0.29607698001480387,
-      "eval_loss": 0.7986361980438232,
-      "eval_runtime": 5.6695,
-      "eval_samples_per_second": 22.577,
-      "eval_steps_per_second": 1.411,
+      "eval_loss": 0.80430006980896,
+      "eval_runtime": 5.8595,
+      "eval_samples_per_second": 21.845,
+      "eval_steps_per_second": 1.365,
       "step": 100
     },
     {
       "epoch": 0.31088082901554404,
-      "grad_norm": 0.4975713789463043,
+      "grad_norm": 0.12097267806529999,
       "learning_rate": 1.741059911251997e-05,
-      "loss": 0.7731,
+      "loss": 0.7786,
       "step": 105
     },
     {
       "epoch": 0.32568467801628426,
-      "grad_norm": 0.5265988707542419,
+      "grad_norm": 0.12593290209770203,
       "learning_rate": 1.7052715352713076e-05,
-      "loss": 0.7674,
+      "loss": 0.7727,
       "step": 110
     },
     {
       "epoch": 0.3404885270170244,
-      "grad_norm": 0.5293793082237244,
+      "grad_norm": 0.12331051379442215,
       "learning_rate": 1.667588140216154e-05,
-      "loss": 0.7943,
+      "loss": 0.7994,
       "step": 115
     },
     {
       "epoch": 0.35529237601776464,
-      "grad_norm": 0.5241897702217102,
+      "grad_norm": 0.1359540820121765,
       "learning_rate": 1.628110978935756e-05,
-      "loss": 0.7687,
+      "loss": 0.774,
       "step": 120
     },
     {
       "epoch": 0.3700962250185048,
-      "grad_norm": 0.537781834602356,
+      "grad_norm": 0.13382965326309204,
       "learning_rate": 1.586946124013354e-05,
-      "loss": 0.7683,
+      "loss": 0.7734,
       "step": 125
     },
     {
       "epoch": 0.38490007401924503,
-      "grad_norm": 0.5170732736587524,
+      "grad_norm": 0.12126770615577698,
       "learning_rate": 1.5442041827560274e-05,
-      "loss": 0.7449,
+      "loss": 0.7497,
       "step": 130
     },
     {
       "epoch": 0.3997039230199852,
-      "grad_norm": 0.5464586019515991,
+      "grad_norm": 0.1166161447763443,
       "learning_rate": 1.5000000000000002e-05,
-      "loss": 0.7558,
+      "loss": 0.7607,
       "step": 135
     },
     {
       "epoch": 0.41450777202072536,
-      "grad_norm": 0.5261128544807434,
+      "grad_norm": 0.12959471344947815,
       "learning_rate": 1.4544523495299843e-05,
-      "loss": 0.762,
+      "loss": 0.7669,
       "step": 140
     },
     {
       "epoch": 0.4293116210214656,
-      "grad_norm": 0.565876305103302,
+      "grad_norm": 0.134933203458786,
       "learning_rate": 1.4076836149416889e-05,
-      "loss": 0.778,
+      "loss": 0.7829,
       "step": 145
     },
     {
       "epoch": 0.44411547002220575,
-      "grad_norm": 0.5250110030174255,
+      "grad_norm": 0.13227751851081848,
       "learning_rate": 1.3598194608050011e-05,
-      "loss": 0.7627,
+      "loss": 0.7677,
       "step": 150
     },
     {
       "epoch": 0.45891931902294597,
-      "grad_norm": 0.47735342383384705,
+      "grad_norm": 0.11633738875389099,
       "learning_rate": 1.3109884950114007e-05,
-      "loss": 0.7521,
+      "loss": 0.7567,
       "step": 155
     },
     {
       "epoch": 0.47372316802368614,
-      "grad_norm": 0.4897947609424591,
+      "grad_norm": 0.12226972728967667,
       "learning_rate": 1.2613219232128608e-05,
-      "loss": 0.752,
+      "loss": 0.7568,
       "step": 160
     },
     {
       "epoch": 0.48852701702442636,
-      "grad_norm": 0.49062302708625793,
+      "grad_norm": 0.11851673573255539,
       "learning_rate": 1.2109531962807333e-05,
-      "loss": 0.7536,
+      "loss": 0.7583,
       "step": 165
     },
     {
       "epoch": 0.5033308660251665,
-      "grad_norm": 0.47276487946510315,
+      "grad_norm": 0.118564672768116,
       "learning_rate": 1.1600176517318742e-05,
-      "loss": 0.7581,
+      "loss": 0.7631,
       "step": 170
     },
     {
       "epoch": 0.5181347150259067,
-      "grad_norm": 0.5006165504455566,
+      "grad_norm": 0.12198054790496826,
       "learning_rate": 1.1086521500854746e-05,
-      "loss": 0.7452,
+      "loss": 0.75,
       "step": 175
     },
     {
       "epoch": 0.532938564026647,
-      "grad_norm": 0.49434661865234375,
+      "grad_norm": 0.1204327940940857,
       "learning_rate": 1.0569947071276847e-05,
-      "loss": 0.766,
+      "loss": 0.7708,
       "step": 180
     },
     {
       "epoch": 0.5477424130273871,
-      "grad_norm": 0.5508667826652527,
+      "grad_norm": 0.13036802411079407,
       "learning_rate": 1.0051841230721065e-05,
-      "loss": 0.759,
+      "loss": 0.764,
       "step": 185
     },
     {
       "epoch": 0.5625462620281273,
-      "grad_norm": 0.49814572930336,
+      "grad_norm": 0.13102592527866364,
       "learning_rate": 9.533596096125826e-06,
-      "loss": 0.7656,
+      "loss": 0.7705,
       "step": 190
     },
     {
       "epoch": 0.5773501110288675,
-      "grad_norm": 0.48042887449264526,
+      "grad_norm": 0.12162639945745468,
       "learning_rate": 9.016604158703654e-06,
-      "loss": 0.7397,
+      "loss": 0.7444,
       "step": 195
     },
     {
       "epoch": 0.5921539600296077,
-      "grad_norm": 0.4838988482952118,
+      "grad_norm": 0.13102519512176514,
       "learning_rate": 8.502254542407186e-06,
-      "loss": 0.7375,
+      "loss": 0.7423,
       "step": 200
     },
     {
       "epoch": 0.5921539600296077,
-      "eval_loss": 0.7734147310256958,
-      "eval_runtime": 5.714,
-      "eval_samples_per_second": 22.401,
-      "eval_steps_per_second": 1.4,
+      "eval_loss": 0.7782641053199768,
+      "eval_runtime": 5.9255,
+      "eval_samples_per_second": 21.601,
+      "eval_steps_per_second": 1.35,
       "step": 200
     },
     {
       "epoch": 0.6069578090303479,
-      "grad_norm": 0.47851860523223877,
+      "grad_norm": 0.11822319775819778,
       "learning_rate": 7.991929271442817e-06,
-      "loss": 0.7415,
+      "loss": 0.7461,
       "step": 205
     },
     {
       "epoch": 0.6217616580310881,
-      "grad_norm": 0.49189141392707825,
+      "grad_norm": 0.11643572896718979,
       "learning_rate": 7.48699955686089e-06,
-      "loss": 0.7437,
+      "loss": 0.7483,
       "step": 210
     },
     {
       "epoch": 0.6365655070318282,
-      "grad_norm": 0.4642721116542816,
+      "grad_norm": 0.11460445076227188,
       "learning_rate": 6.988822112200157e-06,
-      "loss": 0.7517,
+      "loss": 0.7567,
       "step": 215
     },
     {
       "epoch": 0.6513693560325685,
-      "grad_norm": 0.5027055144309998,
+      "grad_norm": 0.12710332870483398,
       "learning_rate": 6.498735508086094e-06,
-      "loss": 0.755,
+      "loss": 0.7597,
       "step": 220
     },
     {
       "epoch": 0.6661732050333087,
-      "grad_norm": 0.44715380668640137,
+      "grad_norm": 0.11362723261117935,
       "learning_rate": 6.018056575578075e-06,
-      "loss": 0.7487,
+      "loss": 0.7537,
       "step": 225
     },
     {
       "epoch": 0.6809770540340488,
-      "grad_norm": 0.41352778673171997,
+      "grad_norm": 0.10726357251405716,
       "learning_rate": 5.548076867929331e-06,
-      "loss": 0.7455,
+      "loss": 0.7503,
       "step": 230
     },
     {
       "epoch": 0.695780903034789,
-      "grad_norm": 0.4372292757034302,
+      "grad_norm": 0.11590282618999481,
       "learning_rate": 5.090059190266779e-06,
-      "loss": 0.7336,
+      "loss": 0.7384,
       "step": 235
     },
     {
       "epoch": 0.7105847520355293,
-      "grad_norm": 0.43178072571754456,
+      "grad_norm": 0.10790540277957916,
       "learning_rate": 4.645234206515171e-06,
-      "loss": 0.7387,
+      "loss": 0.7436,
       "step": 240
     },
     {
       "epoch": 0.7253886010362695,
-      "grad_norm": 0.4522764980792999,
+      "grad_norm": 0.11345735192298889,
       "learning_rate": 4.214797132682597e-06,
-      "loss": 0.7352,
+      "loss": 0.7401,
       "step": 245
     },
     {
       "epoch": 0.7401924500370096,
-      "grad_norm": 0.4542970359325409,
+      "grad_norm": 0.11736641824245453,
       "learning_rate": 3.799904525392251e-06,
-      "loss": 0.7421,
+      "loss": 0.747,
       "step": 250
     },
     {
       "epoch": 0.7549962990377498,
-      "grad_norm": 0.443574458360672,
+      "grad_norm": 0.11365947127342224,
       "learning_rate": 3.401671174289469e-06,
-      "loss": 0.7323,
+      "loss": 0.7371,
       "step": 255
     },
     {
       "epoch": 0.7698001480384901,
-      "grad_norm": 0.4417092502117157,
+      "grad_norm": 0.11066755652427673,
       "learning_rate": 3.021167106673928e-06,
-      "loss": 0.7483,
+      "loss": 0.7531,
       "step": 260
     },
     {
       "epoch": 0.7846039970392302,
-      "grad_norm": 0.435789555311203,
+      "grad_norm": 0.10806834697723389,
       "learning_rate": 2.6594147124053983e-06,
-      "loss": 0.7371,
+      "loss": 0.742,
       "step": 265
     },
     {
       "epoch": 0.7994078460399704,
-      "grad_norm": 0.4441350996494293,
+      "grad_norm": 0.1102728471159935,
       "learning_rate": 2.317385996808195e-06,
-      "loss": 0.7488,
+      "loss": 0.7536,
       "step": 270
     },
     {
       "epoch": 0.8142116950407106,
-      "grad_norm": 0.41490909457206726,
+      "grad_norm": 0.10305000841617584,
       "learning_rate": 1.9959999689556407e-06,
-      "loss": 0.7413,
+      "loss": 0.7463,
       "step": 275
     },
     {
       "epoch": 0.8290155440414507,
-      "grad_norm": 0.4214578866958618,
+      "grad_norm": 0.10573720186948776,
       "learning_rate": 1.6961201723520248e-06,
-      "loss": 0.7272,
+      "loss": 0.732,
       "step": 280
     },
     {
       "epoch": 0.843819393042191,
-      "grad_norm": 0.4023810923099518,
+      "grad_norm": 0.10039414465427399,
       "learning_rate": 1.4185523646469822e-06,
-      "loss": 0.752,
+      "loss": 0.757,
       "step": 285
     },
     {
       "epoch": 0.8586232420429312,
-      "grad_norm": 0.4226447641849518,
+      "grad_norm": 0.10779191553592682,
       "learning_rate": 1.1640423526166987e-06,
-      "loss": 0.7298,
+      "loss": 0.7348,
       "step": 290
     },
     {
       "epoch": 0.8734270910436713,
-      "grad_norm": 0.405353307723999,
+      "grad_norm": 0.1016748696565628,
       "learning_rate": 9.332739882292752e-07,
-      "loss": 0.7559,
+      "loss": 0.7608,
       "step": 295
     },
     {
       "epoch": 0.8882309400444115,
-      "grad_norm": 0.4032580256462097,
+      "grad_norm": 0.10107531398534775,
       "learning_rate": 7.268673311786378e-07,
-      "loss": 0.7455,
+      "loss": 0.7507,
       "step": 300
     },
     {
       "epoch": 0.8882309400444115,
-      "eval_loss": 0.7648502588272095,
-      "eval_runtime": 5.5578,
-      "eval_samples_per_second": 23.031,
-      "eval_steps_per_second": 1.439,
+      "eval_loss": 0.7698361873626709,
+      "eval_runtime": 5.8048,
+      "eval_samples_per_second": 22.051,
+      "eval_steps_per_second": 1.378,
       "step": 300
     },
     {
       "epoch": 0.9030347890451518,
-      "grad_norm": 0.3890097737312317,
+      "grad_norm": 0.09836234152317047,
       "learning_rate": 5.453769828241872e-07,
-      "loss": 0.7294,
+      "loss": 0.7343,
       "step": 305
     },
     {
       "epoch": 0.9178386380458919,
-      "grad_norm": 0.4142417311668396,
+      "grad_norm": 0.10397884249687195,
       "learning_rate": 3.8929059601275463e-07,
-      "loss": 0.7617,
+      "loss": 0.7668,
       "step": 310
     },
     {
       "epoch": 0.9326424870466321,
-      "grad_norm": 0.3959888219833374,
+      "grad_norm": 0.09893100708723068,
       "learning_rate": 2.5902756478688674e-07,
-      "loss": 0.7442,
+      "loss": 0.749,
       "step": 315
     },
     {
       "epoch": 0.9474463360473723,
-      "grad_norm": 0.39996781945228577,
+      "grad_norm": 0.1005384624004364,
       "learning_rate": 1.5493789750014032e-07,
-      "loss": 0.7459,
+      "loss": 0.7509,
       "step": 320
     },
     {
       "epoch": 0.9622501850481125,
-      "grad_norm": 0.42691072821617126,
+      "grad_norm": 0.10756874084472656,
       "learning_rate": 7.730127636723539e-08,
-      "loss": 0.7266,
+      "loss": 0.7315,
       "step": 325
     },
     {
       "epoch": 0.9770540340488527,
-      "grad_norm": 0.4197230637073517,
+      "grad_norm": 0.1049792617559433,
       "learning_rate": 2.6326305976001054e-08,
-      "loss": 0.7312,
+      "loss": 0.7362,
       "step": 330
     },
     {
       "epoch": 0.9918578830495929,
-      "grad_norm": 0.44191622734069824,
+      "grad_norm": 0.11218578368425369,
       "learning_rate": 2.149952780321485e-09,
-      "loss": 0.7525,
+      "loss": 0.7576,
       "step": 335
     },
     {
       "epoch": 0.997779422649889,
       "step": 337,
       "total_flos": 76745898196992.0,
-      "train_loss": 0.7814551896086787,
-      "train_runtime": 4577.2961,
-      "train_samples_per_second": 4.721,
-      "train_steps_per_second": 0.074
+      "train_loss": 0.7870922902217604,
+      "train_runtime": 4743.2538,
+      "train_samples_per_second": 4.556,
+      "train_steps_per_second": 0.071
     }
   ],
   "logging_steps": 5,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6e01b3694b919c97bbee46918cea0ad569de1f5a7e12fd01da965ead09ddea96
+oid sha256:0516457824250b3c72e8f3cba31e7ed9ce0733070d5423a99e9bb91778da8840
 size 7352