Model save

README.md

@@ -1,11 +1,9 @@
 ---
-base_model: Qwen/Qwen2.5-7B-Instruct
+base_model: Rixhabh/Loong-MATH-train
 library_name: transformers
 model_name: Loong-MATH-train
 tags:
 - generated_from_trainer
-- Loong
-- hendrycks_math
 - trl
 - sft
 licence: license
@@ -13,7 +11,7 @@ licence: license
 
 # Model Card for Loong-MATH-train
 
-This model is a fine-tuned version of [Qwen/Qwen2.5-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct).
+This model is a fine-tuned version of [Rixhabh/Loong-MATH-train](https://huggingface.co/Rixhabh/Loong-MATH-train).
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start
@@ -29,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/yueming-lai-eigent/huggingface/runs/uraus13b) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/yueming-lai-eigent/huggingface/runs/ekwwyi0r) 
 
 
 This model was trained with SFT.

all_results.json

@@ -1,8 +1,8 @@
 {
-    "total_flos": 23526116163584.0,
-    "train_loss": 0.0,
-    "train_runtime": 0.8857,
+    "total_flos": 47054728527872.0,
+    "train_loss": 0.10314766578376293,
+    "train_runtime": 4354.8726,
     "train_samples": 7500,
-    "train_samples_per_second": 3483.055,
-    "train_steps_per_second": 22.581
+    "train_samples_per_second": 1.417,
+    "train_steps_per_second": 0.009
 }

train_results.json

@@ -1,8 +1,8 @@
 {
-    "total_flos": 23526116163584.0,
-    "train_loss": 0.0,
-    "train_runtime": 0.8857,
+    "total_flos": 47054728527872.0,
+    "train_loss": 0.10314766578376293,
+    "train_runtime": 4354.8726,
     "train_samples": 7500,
-    "train_samples_per_second": 3483.055,
-    "train_steps_per_second": 22.581
+    "train_samples_per_second": 1.417,
+    "train_steps_per_second": 0.009
 }

trainer_state.json

@@ -2,9 +2,9 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 4.828478964401294,
+  "epoch": 9.828478964401295,
   "eval_steps": 500,
-  "global_step": 20,
+  "global_step": 40,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -190,19 +190,199 @@
       "step": 20
     },
     {
-      "epoch": 4.828478964401294,
-      "step": 20,
-      "total_flos": 23526116163584.0,
-      "train_loss": 0.0,
-      "train_runtime": 0.8857,
-      "train_samples_per_second": 3483.055,
-      "train_steps_per_second": 22.581
+      "epoch": 5.207119741100324,
+      "grad_norm": 0.29226028229762824,
+      "learning_rate": 0.0,
+      "loss": 0.2581,
+      "mean_token_accuracy": 0.9223490478470922,
+      "num_tokens": 520682.0,
+      "step": 21
+    },
+    {
+      "epoch": 5.414239482200648,
+      "grad_norm": 0.3039506268459287,
+      "learning_rate": 9.5e-06,
+      "loss": 0.2612,
+      "mean_token_accuracy": 0.921809334307909,
+      "num_tokens": 1042545.0,
+      "step": 22
+    },
+    {
+      "epoch": 5.621359223300971,
+      "grad_norm": 0.3292461938787561,
+      "learning_rate": 9e-06,
+      "loss": 0.2338,
+      "mean_token_accuracy": 0.9292812049388885,
+      "num_tokens": 1563697.0,
+      "step": 23
+    },
+    {
+      "epoch": 5.828478964401294,
+      "grad_norm": 0.46420193174977303,
+      "learning_rate": 8.5e-06,
+      "loss": 0.2446,
+      "mean_token_accuracy": 0.9261427698656917,
+      "num_tokens": 2087270.0,
+      "step": 24
+    },
+    {
+      "epoch": 6.207119741100324,
+      "grad_norm": 0.5152631282390204,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 0.4273,
+      "mean_token_accuracy": 0.9354684165545872,
+      "num_tokens": 2942515.0,
+      "step": 25
+    },
+    {
+      "epoch": 6.414239482200648,
+      "grad_norm": 0.397171201247213,
+      "learning_rate": 7.500000000000001e-06,
+      "loss": 0.1971,
+      "mean_token_accuracy": 0.9410504633560777,
+      "num_tokens": 3464980.0,
+      "step": 26
+    },
+    {
+      "epoch": 6.621359223300971,
+      "grad_norm": 0.4038092235036345,
+      "learning_rate": 7e-06,
+      "loss": 0.1936,
+      "mean_token_accuracy": 0.9419528925791383,
+      "num_tokens": 3988553.0,
+      "step": 27
+    },
+    {
+      "epoch": 6.828478964401294,
+      "grad_norm": 0.4383287277331565,
+      "learning_rate": 6.5000000000000004e-06,
+      "loss": 0.1982,
+      "mean_token_accuracy": 0.9402873041108251,
+      "num_tokens": 4512841.0,
+      "step": 28
+    },
+    {
+      "epoch": 7.207119741100324,
+      "grad_norm": 0.5819451162543645,
+      "learning_rate": 6e-06,
+      "loss": 0.3495,
+      "mean_token_accuracy": 0.9478773713111878,
+      "num_tokens": 5363834.0,
+      "step": 29
+    },
+    {
+      "epoch": 7.414239482200648,
+      "grad_norm": 0.35512055236444356,
+      "learning_rate": 5.500000000000001e-06,
+      "loss": 0.1599,
+      "mean_token_accuracy": 0.9522946244105697,
+      "num_tokens": 5887407.0,
+      "step": 30
+    },
+    {
+      "epoch": 7.621359223300971,
+      "grad_norm": 0.6014961197056318,
+      "learning_rate": 5e-06,
+      "loss": 0.1586,
+      "mean_token_accuracy": 0.9526052679866552,
+      "num_tokens": 6408559.0,
+      "step": 31
+    },
+    {
+      "epoch": 7.828478964401294,
+      "grad_norm": 0.4044887603769801,
+      "learning_rate": 4.5e-06,
+      "loss": 0.1515,
+      "mean_token_accuracy": 0.9546759780496359,
+      "num_tokens": 6930887.0,
+      "step": 32
+    },
+    {
+      "epoch": 8.207119741100323,
+      "grad_norm": 0.527657049732786,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 0.2954,
+      "mean_token_accuracy": 0.9571067219688779,
+      "num_tokens": 7784698.0,
+      "step": 33
+    },
+    {
+      "epoch": 8.414239482200648,
+      "grad_norm": 0.5655954589470128,
+      "learning_rate": 3.5e-06,
+      "loss": 0.1301,
+      "mean_token_accuracy": 0.9615885661914945,
+      "num_tokens": 8307878.0,
+      "step": 34
+    },
+    {
+      "epoch": 8.62135922330097,
+      "grad_norm": 0.3905499777914469,
+      "learning_rate": 3e-06,
+      "loss": 0.1309,
+      "mean_token_accuracy": 0.9616770427674055,
+      "num_tokens": 8831564.0,
+      "step": 35
+    },
+    {
+      "epoch": 8.828478964401295,
+      "grad_norm": 0.3609762708880154,
+      "learning_rate": 2.5e-06,
+      "loss": 0.1301,
+      "mean_token_accuracy": 0.961113647557795,
+      "num_tokens": 9354489.0,
+      "step": 36
+    },
+    {
+      "epoch": 9.207119741100323,
+      "grad_norm": 0.6575328899879626,
+      "learning_rate": 2.0000000000000003e-06,
+      "loss": 0.2498,
+      "mean_token_accuracy": 0.9636295852207002,
+      "num_tokens": 10210406.0,
+      "step": 37
+    },
+    {
+      "epoch": 9.414239482200648,
+      "grad_norm": 0.28343130576610354,
+      "learning_rate": 1.5e-06,
+      "loss": 0.1245,
+      "mean_token_accuracy": 0.9638419672846794,
+      "num_tokens": 10729521.0,
+      "step": 38
+    },
+    {
+      "epoch": 9.62135922330097,
+      "grad_norm": 0.3576238970586715,
+      "learning_rate": 1.0000000000000002e-06,
+      "loss": 0.1181,
+      "mean_token_accuracy": 0.965806245803833,
+      "num_tokens": 11253094.0,
+      "step": 39
+    },
+    {
+      "epoch": 9.828478964401295,
+      "grad_norm": 0.38762707391762197,
+      "learning_rate": 5.000000000000001e-07,
+      "loss": 0.1138,
+      "mean_token_accuracy": 0.9672208018600941,
+      "num_tokens": 11773251.0,
+      "step": 40
+    },
+    {
+      "epoch": 9.828478964401295,
+      "step": 40,
+      "total_flos": 47054728527872.0,
+      "train_loss": 0.10314766578376293,
+      "train_runtime": 4354.8726,
+      "train_samples_per_second": 1.417,
+      "train_steps_per_second": 0.009
     }
   ],
   "logging_steps": 1.0,
-  "max_steps": 20,
+  "max_steps": 40,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 5,
+  "num_train_epochs": 10,
   "save_steps": 10,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -216,7 +396,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 23526116163584.0,
+  "total_flos": 47054728527872.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null