Model save

README.md

@@ -1,11 +1,9 @@
 ---
 base_model: Qwen/Qwen2.5-1.5B-Instruct
-datasets: zwt963/FinReasoner
 library_name: transformers
 model_name: Qwen2.5-1.5B-Instruct-Open-R1-GRPO
 tags:
 - generated_from_trainer
-- open-r1
 - trl
 - grpo
 licence: license
@@ -13,7 +11,7 @@ licence: license
 
 # Model Card for Qwen2.5-1.5B-Instruct-Open-R1-GRPO
 
-This model is a fine-tuned version of [Qwen/Qwen2.5-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct) on the [zwt963/FinReasoner](https://huggingface.co/datasets/zwt963/FinReasoner) dataset.
+This model is a fine-tuned version of [Qwen/Qwen2.5-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct).
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start
@@ -29,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/zwt963/huggingface/runs/qf6r4ry3) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/zwt963/huggingface/runs/z82h6cyg) 
 
 
 This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).

all_results.json

@@ -5,9 +5,9 @@
     "eval_samples_per_second": 7.75,
     "eval_steps_per_second": 0.278,
     "total_flos": 0.0,
-    "train_loss": 0.007735961908474564,
-    "train_runtime": 2204.9615,
+    "train_loss": 0.03456688816659152,
+    "train_runtime": 2502.5824,
     "train_samples": 891,
-    "train_samples_per_second": 4.041,
-    "train_steps_per_second": 0.036
+    "train_samples_per_second": 3.56,
+    "train_steps_per_second": 0.032
 }

config.json

@@ -23,7 +23,7 @@
   "tie_word_embeddings": true,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.49.0.dev0",
-  "use_cache": true,
+  "use_cache": false,
   "use_sliding_window": false,
   "vocab_size": 151936
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bcd36eb3e7cb0d3f525d6098036ca449aaf11a2be8091ebc57e69010cb9f99ab
+oid sha256:feab802ea5480be54e59770454989982c4a4ed440afd8de60350a365f1119295
 size 3554214752

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.007735961908474564,
-    "train_runtime": 2204.9615,
+    "train_loss": 0.03456688816659152,
+    "train_runtime": 2502.5824,
     "train_samples": 891,
-    "train_samples_per_second": 4.041,
-    "train_steps_per_second": 0.036
+    "train_samples_per_second": 3.56,
+    "train_steps_per_second": 0.032
 }

trainer_state.json

@@ -9,229 +9,229 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 91.19632097482682,
+      "completion_length": 105.54286189079285,
       "epoch": 1.25,
-      "grad_norm": 1.1969395875930786,
-      "kl": 0.1442023515701294,
+      "grad_norm": 2.6259748935699463,
+      "kl": 0.09056792259216309,
       "learning_rate": 1.9961946980917457e-05,
-      "loss": 0.0211,
-      "reward": 1.1485491564497352,
-      "reward_std": 0.3577419674023986,
-      "rewards/accuracy_reward": 0.3771205507218838,
-      "rewards/format_reward": 0.7714285938069224,
+      "loss": 0.0036,
+      "reward": 1.1364955766126514,
+      "reward_std": 0.41473900550045073,
+      "rewards/accuracy_reward": 0.3872768019326031,
+      "rewards/format_reward": 0.7492187711410224,
       "step": 10
     },
     {
       "epoch": 1.25,
-      "eval_completion_length": 41.17280626296997,
-      "eval_kl": 9.25634765625,
-      "eval_loss": 0.3707638382911682,
-      "eval_reward": 1.5239955931901932,
-      "eval_reward_std": 0.11370187811553478,
-      "eval_rewards/accuracy_reward": 0.5267857350409031,
-      "eval_rewards/format_reward": 0.997209832072258,
-      "eval_runtime": 17.0057,
-      "eval_samples_per_second": 13.113,
-      "eval_steps_per_second": 0.47,
+      "eval_completion_length": 63.21034049987793,
+      "eval_kl": 32.33154296875,
+      "eval_loss": 1.2949256896972656,
+      "eval_reward": 1.5097140073776245,
+      "eval_reward_std": 0.20717254094779491,
+      "eval_rewards/accuracy_reward": 0.5242228880524635,
+      "eval_rewards/format_reward": 0.9854911118745804,
+      "eval_runtime": 48.0317,
+      "eval_samples_per_second": 4.643,
+      "eval_steps_per_second": 0.167,
       "step": 10
     },
     {
-      "completion_length": 77.98214609622956,
+      "completion_length": 69.01763689517975,
       "epoch": 2.5,
-      "grad_norm": 0.38055306673049927,
-      "kl": 0.1487640380859375,
+      "grad_norm": 0.5660845637321472,
+      "kl": 0.1720977783203125,
       "learning_rate": 1.866025403784439e-05,
-      "loss": 0.0059,
-      "reward": 1.5377232797443867,
-      "reward_std": 0.112967933062464,
-      "rewards/accuracy_reward": 0.5417410933063366,
-      "rewards/format_reward": 0.9959821529686451,
+      "loss": 0.0069,
+      "reward": 1.5524554237723351,
+      "reward_std": 0.1543258734047413,
+      "rewards/accuracy_reward": 0.5655134161934257,
+      "rewards/format_reward": 0.9869419772177934,
       "step": 20
     },
     {
       "epoch": 2.5,
-      "eval_completion_length": 74.35160541534424,
-      "eval_kl": 79363.1533203125,
-      "eval_loss": 3201.5810546875,
-      "eval_reward": 1.4969618618488312,
-      "eval_reward_std": 0.11330691166222095,
-      "eval_rewards/accuracy_reward": 0.4997520036995411,
-      "eval_rewards/format_reward": 0.997209832072258,
-      "eval_runtime": 16.492,
-      "eval_samples_per_second": 13.522,
-      "eval_steps_per_second": 0.485,
+      "eval_completion_length": 94.81211185455322,
+      "eval_kl": 2333184.209838867,
+      "eval_loss": 93653.515625,
+      "eval_reward": 1.4855117946863174,
+      "eval_reward_std": 0.1817244552075863,
+      "eval_rewards/accuracy_reward": 0.49109210819005966,
+      "eval_rewards/format_reward": 0.994419664144516,
+      "eval_runtime": 22.2095,
+      "eval_samples_per_second": 10.041,
+      "eval_steps_per_second": 0.36,
       "step": 20
     },
     {
-      "completion_length": 68.95022611618042,
+      "completion_length": 102.01250414848327,
       "epoch": 3.75,
-      "grad_norm": 0.411288321018219,
-      "kl": 0.2060882568359375,
+      "grad_norm": 0.38660958409309387,
+      "kl": 0.16029052734375,
       "learning_rate": 1.573576436351046e-05,
-      "loss": 0.0081,
-      "reward": 1.601116143167019,
-      "reward_std": 0.09683309164829552,
-      "rewards/accuracy_reward": 0.6053571676835418,
-      "rewards/format_reward": 0.9957589406520129,
+      "loss": 0.0064,
+      "reward": 1.6341518580913543,
+      "reward_std": 0.12228572466410696,
+      "rewards/accuracy_reward": 0.6387277023866773,
+      "rewards/format_reward": 0.9954241190105677,
       "step": 30
     },
     {
       "epoch": 3.75,
-      "eval_completion_length": 80.51895523071289,
-      "eval_kl": 1.675048828125,
-      "eval_loss": 0.06678587198257446,
-      "eval_reward": 1.5466683954000473,
-      "eval_reward_std": 0.08276130817830563,
-      "eval_rewards/accuracy_reward": 0.5472263805568218,
-      "eval_rewards/format_reward": 0.9994419664144516,
-      "eval_runtime": 18.4204,
-      "eval_samples_per_second": 12.106,
-      "eval_steps_per_second": 0.434,
+      "eval_completion_length": 84.20744895935059,
+      "eval_kl": 700088320.0,
+      "eval_loss": 28113970.0,
+      "eval_reward": 1.5764509439468384,
+      "eval_reward_std": 0.08258435316383839,
+      "eval_rewards/accuracy_reward": 0.5803571604192257,
+      "eval_rewards/format_reward": 0.9960937649011612,
+      "eval_runtime": 19.0164,
+      "eval_samples_per_second": 11.727,
+      "eval_steps_per_second": 0.421,
       "step": 30
     },
     {
-      "completion_length": 86.34196748733521,
+      "completion_length": 80.08147611618043,
       "epoch": 5.0,
-      "grad_norm": 0.30611079931259155,
-      "kl": 0.15224609375,
+      "grad_norm": 0.600352942943573,
+      "kl": 0.1693634033203125,
       "learning_rate": 1.1736481776669307e-05,
-      "loss": 0.006,
-      "reward": 1.7011161364614964,
-      "reward_std": 0.05814310675486922,
-      "rewards/accuracy_reward": 0.7026785946451127,
-      "rewards/format_reward": 0.9984375044703484,
+      "loss": 0.0067,
+      "reward": 1.7243304178118706,
+      "reward_std": 0.051013148529455064,
+      "rewards/accuracy_reward": 0.726674135029316,
+      "rewards/format_reward": 0.9976562574505806,
       "step": 40
     },
     {
       "epoch": 5.0,
-      "eval_completion_length": 95.69973850250244,
-      "eval_kl": 1.921630859375,
-      "eval_loss": 0.07705733180046082,
-      "eval_reward": 1.5726687014102936,
-      "eval_reward_std": 0.08731811679899693,
-      "eval_rewards/accuracy_reward": 0.5726686716079712,
-      "eval_rewards/format_reward": 1.0,
-      "eval_runtime": 27.6097,
-      "eval_samples_per_second": 8.077,
-      "eval_steps_per_second": 0.29,
+      "eval_completion_length": 77.53683376312256,
+      "eval_kl": 341368621.0,
+      "eval_loss": 13754997.0,
+      "eval_reward": 1.5658482760190964,
+      "eval_reward_std": 0.06126835383474827,
+      "eval_rewards/accuracy_reward": 0.5691964514553547,
+      "eval_rewards/format_reward": 0.9966517984867096,
+      "eval_runtime": 24.4064,
+      "eval_samples_per_second": 9.137,
+      "eval_steps_per_second": 0.328,
       "step": 40
     },
     {
-      "completion_length": 88.07578473091125,
+      "completion_length": 80.83471281528473,
       "epoch": 6.25,
-      "grad_norm": 0.186005100607872,
-      "kl": 0.138330078125,
+      "grad_norm": 0.4441774785518646,
+      "kl": 0.16822509765625,
       "learning_rate": 7.411809548974792e-06,
-      "loss": 0.0056,
-      "reward": 1.7131697207689285,
-      "reward_std": 0.046526999399065974,
-      "rewards/accuracy_reward": 0.7152902040630579,
-      "rewards/format_reward": 0.997879472374916,
+      "loss": 0.0068,
+      "reward": 1.7502232857048512,
+      "reward_std": 0.04045138210058212,
+      "rewards/accuracy_reward": 0.7536830637603998,
+      "rewards/format_reward": 0.9965401917696,
       "step": 50
     },
     {
       "epoch": 6.25,
-      "eval_completion_length": 82.68043422698975,
-      "eval_kl": 0.50927734375,
-      "eval_loss": 0.020424507558345795,
-      "eval_reward": 1.5327174961566925,
-      "eval_reward_std": 0.06632188148796558,
-      "eval_rewards/accuracy_reward": 0.5343915484845638,
-      "eval_rewards/format_reward": 0.9983258992433548,
-      "eval_runtime": 22.0694,
-      "eval_samples_per_second": 10.104,
-      "eval_steps_per_second": 0.362,
+      "eval_completion_length": 81.76986408233643,
+      "eval_kl": 230830211.78515625,
+      "eval_loss": 9269680.0,
+      "eval_reward": 1.5467717200517654,
+      "eval_reward_std": 0.10743661969900131,
+      "eval_rewards/accuracy_reward": 0.5506779365241528,
+      "eval_rewards/format_reward": 0.9960937649011612,
+      "eval_runtime": 21.5108,
+      "eval_samples_per_second": 10.367,
+      "eval_steps_per_second": 0.372,
       "step": 50
     },
     {
-      "completion_length": 82.71183357238769,
+      "completion_length": 79.44810581207275,
       "epoch": 7.5,
-      "grad_norm": 0.21738964319229126,
-      "kl": 0.133831787109375,
+      "grad_norm": 0.18081872165203094,
+      "kl": 0.1697906494140625,
       "learning_rate": 3.5721239031346067e-06,
-      "loss": 0.0054,
-      "reward": 1.7395090013742447,
-      "reward_std": 0.03326362115330994,
-      "rewards/accuracy_reward": 0.7408482428640127,
-      "rewards/format_reward": 0.9986607193946838,
+      "loss": 0.0068,
+      "reward": 1.7613840006291865,
+      "reward_std": 0.04446553010493517,
+      "rewards/accuracy_reward": 0.7638393169268965,
+      "rewards/format_reward": 0.997544652223587,
       "step": 60
     },
     {
       "epoch": 7.5,
-      "eval_completion_length": 85.38940906524658,
-      "eval_kl": 0.82177734375,
-      "eval_loss": 0.033002614974975586,
-      "eval_reward": 1.538297861814499,
-      "eval_reward_std": 0.053902728483080864,
-      "eval_rewards/accuracy_reward": 0.5399719178676605,
-      "eval_rewards/format_reward": 0.9983258992433548,
-      "eval_runtime": 24.6306,
-      "eval_samples_per_second": 9.054,
-      "eval_steps_per_second": 0.325,
+      "eval_completion_length": 81.85964775085449,
+      "eval_kl": 35744152.900390625,
+      "eval_loss": 1435289.125,
+      "eval_reward": 1.5892857760190964,
+      "eval_reward_std": 0.0803372673690319,
+      "eval_rewards/accuracy_reward": 0.5931919813156128,
+      "eval_rewards/format_reward": 0.9960937649011612,
+      "eval_runtime": 45.193,
+      "eval_samples_per_second": 4.934,
+      "eval_steps_per_second": 0.177,
       "step": 60
     },
     {
-      "completion_length": 87.68672251701355,
+      "completion_length": 73.37768139839173,
       "epoch": 8.75,
-      "grad_norm": 0.16990460455417633,
-      "kl": 0.12266845703125,
+      "grad_norm": 1514.581298828125,
+      "kl": 5.606082153320313,
       "learning_rate": 9.369221296335007e-07,
-      "loss": 0.0049,
-      "reward": 1.7386161454021931,
-      "reward_std": 0.025158090936020016,
-      "rewards/accuracy_reward": 0.739843780361116,
-      "rewards/format_reward": 0.9987723261117936,
+      "loss": 0.2239,
+      "reward": 1.7611607864499093,
+      "reward_std": 0.046963289054110645,
+      "rewards/accuracy_reward": 0.7637277090921998,
+      "rewards/format_reward": 0.9974330447614193,
       "step": 70
     },
     {
       "epoch": 8.75,
-      "eval_completion_length": 88.4438066482544,
-      "eval_kl": 1.0899658203125,
-      "eval_loss": 0.043835245072841644,
-      "eval_reward": 1.5355076640844345,
-      "eval_reward_std": 0.05012673186138272,
-      "eval_rewards/accuracy_reward": 0.5355076231062412,
-      "eval_rewards/format_reward": 1.0,
-      "eval_runtime": 23.079,
-      "eval_samples_per_second": 9.662,
-      "eval_steps_per_second": 0.347,
+      "eval_completion_length": 74.76281642913818,
+      "eval_kl": 19875971.58935547,
+      "eval_loss": 796283.5625,
+      "eval_reward": 1.6049107760190964,
+      "eval_reward_std": 0.08538927137851715,
+      "eval_rewards/accuracy_reward": 0.6077009104192257,
+      "eval_rewards/format_reward": 0.997209832072258,
+      "eval_runtime": 24.3263,
+      "eval_samples_per_second": 9.167,
+      "eval_steps_per_second": 0.329,
       "step": 70
     },
     {
-      "completion_length": 87.12489142417908,
+      "completion_length": 72.58817234039307,
       "epoch": 10.0,
-      "grad_norm": 0.18827787041664124,
-      "kl": 0.1228912353515625,
+      "grad_norm": 0.32780376076698303,
+      "kl": 0.3870269775390625,
       "learning_rate": 0.0,
-      "loss": 0.0049,
-      "reward": 1.7418527446687222,
-      "reward_std": 0.028121343860402704,
-      "rewards/accuracy_reward": 0.7435268083587289,
-      "rewards/format_reward": 0.9983258992433548,
+      "loss": 0.0155,
+      "reward": 1.7863839961588384,
+      "reward_std": 0.03610877096652985,
+      "rewards/accuracy_reward": 0.7883928820490838,
+      "rewards/format_reward": 0.9979910783469677,
       "step": 80
     },
     {
       "epoch": 10.0,
-      "eval_completion_length": 89.6388931274414,
-      "eval_kl": 0.638916015625,
-      "eval_loss": 0.025608114898204803,
-      "eval_reward": 1.533275529742241,
-      "eval_reward_std": 0.07100645918399096,
-      "eval_rewards/accuracy_reward": 0.5360656566917896,
-      "eval_rewards/format_reward": 0.997209832072258,
-      "eval_runtime": 26.5482,
-      "eval_samples_per_second": 8.4,
-      "eval_steps_per_second": 0.301,
+      "eval_completion_length": 74.61425590515137,
+      "eval_kl": 73859272.65722656,
+      "eval_loss": 2970804.5,
+      "eval_reward": 1.5872809886932373,
+      "eval_reward_std": 0.08278952026739717,
+      "eval_rewards/accuracy_reward": 0.5935846753418446,
+      "eval_rewards/format_reward": 0.9936962723731995,
+      "eval_runtime": 19.1672,
+      "eval_samples_per_second": 11.634,
+      "eval_steps_per_second": 0.417,
       "step": 80
     },
     {
       "epoch": 10.0,
       "step": 80,
       "total_flos": 0.0,
-      "train_loss": 0.007735961908474564,
-      "train_runtime": 2204.9615,
-      "train_samples_per_second": 4.041,
-      "train_steps_per_second": 0.036
+      "train_loss": 0.03456688816659152,
+      "train_runtime": 2502.5824,
+      "train_samples_per_second": 3.56,
+      "train_steps_per_second": 0.032
     }
   ],
   "logging_steps": 10,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7997b82e14676fde33a3d08444edb9eaa791adab6c2582967d3283951b34decf
+oid sha256:40335af4c1c3bf3a5c088c8646e1eb73a4804afe2a7404ba98ef56e9bef84fc6
 size 7224