Model save

README.md

@@ -1,11 +1,9 @@
 ---
-base_model: deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B
-datasets: Blancy/1ktestfrom10kwithdifficultyclasses
+base_model: Qwen/Qwen3-1.7B
 library_name: transformers
 model_name: Qwen2.5-1.5B-Open-R1-GRPO
 tags:
 - generated_from_trainer
-- open-r1
 - trl
 - grpo
 licence: license
@@ -13,7 +11,7 @@ licence: license
 
 # Model Card for Qwen2.5-1.5B-Open-R1-GRPO
 
-This model is a fine-tuned version of [deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B) on the [Blancy/1ktestfrom10kwithdifficultyclasses](https://huggingface.co/datasets/Blancy/1ktestfrom10kwithdifficultyclasses) dataset.
+This model is a fine-tuned version of [Qwen/Qwen3-1.7B](https://huggingface.co/Qwen/Qwen3-1.7B).
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start
@@ -29,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/224015062-chinese-university-of-hong-kong-shenzhen/huggingface/runs/hy313twm) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/224015062-chinese-university-of-hong-kong-shenzhen/huggingface/runs/lb954dij) 
 
 
 This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.004263622482312219,
-    "train_runtime": 9387.6074,
+    "train_loss": 0.013564518235874741,
+    "train_runtime": 16822.299,
     "train_samples": 1000,
-    "train_samples_per_second": 0.107,
-    "train_steps_per_second": 0.004
+    "train_samples_per_second": 0.059,
+    "train_steps_per_second": 0.002
 }

generation_config.json

@@ -1,9 +1,13 @@
 {
-  "_from_model_config": true,
-  "bos_token_id": 151646,
+  "bos_token_id": 151643,
   "do_sample": true,
-  "eos_token_id": 151643,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
   "temperature": 0.6,
+  "top_k": 20,
   "top_p": 0.95,
   "transformers_version": "4.52.3"
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.004263622482312219,
-    "train_runtime": 9387.6074,
+    "train_loss": 0.013564518235874741,
+    "train_runtime": 16822.299,
     "train_samples": 1000,
-    "train_samples_per_second": 0.107,
-    "train_steps_per_second": 0.004
+    "train_samples_per_second": 0.059,
+    "train_steps_per_second": 0.002
 }

trainer_state.json

@@ -10,517 +10,517 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 3006.3005981445312,
+      "completion_length": 3466.5059814453125,
       "epoch": 0.027972027972027972,
-      "grad_norm": 0.5744067430496216,
+      "grad_norm": 0.8895137310028076,
       "kl": 0.0,
       "learning_rate": 0.0,
       "loss": 0.0,
-      "reward": 0.2819940522313118,
-      "reward_std": 0.22714678570628166,
-      "rewards/accuracy_reward": 0.0714285729918629,
+      "reward": 0.4657738283276558,
+      "reward_std": 0.11145408265292645,
+      "rewards/accuracy_reward": 0.08630952658131719,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.210565485060215,
+      "rewards/tag_count_reward": 0.379464291036129,
       "step": 1
     },
     {
-      "completion_length": 3101.8661499023438,
+      "completion_length": 3675.1995239257812,
       "epoch": 0.055944055944055944,
-      "grad_norm": 0.17433099448680878,
+      "grad_norm": 0.545717716217041,
       "kl": 0.0,
       "learning_rate": 5e-06,
       "loss": 0.0,
-      "reward": 0.2872023843228817,
-      "reward_std": 0.26565564423799515,
-      "rewards/accuracy_reward": 0.09226190764456987,
+      "reward": 0.473214291036129,
+      "reward_std": 0.16795706376433372,
+      "rewards/accuracy_reward": 0.09523809934034944,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.1949404813349247,
+      "rewards/tag_count_reward": 0.377976194024086,
       "step": 2
     },
     {
-      "completion_length": 2855.636962890625,
+      "completion_length": 3419.232177734375,
       "epoch": 0.08391608391608392,
-      "grad_norm": 1.1990246772766113,
-      "kl": 0.00012302398681640625,
+      "grad_norm": 0.5103467106819153,
+      "kl": 0.0010442733764648438,
       "learning_rate": 1e-05,
       "loss": 0.0,
-      "reward": 0.3303571417927742,
-      "reward_std": 0.26726851612329483,
-      "rewards/accuracy_reward": 0.10416667093522847,
+      "reward": 0.5677083432674408,
+      "reward_std": 0.17301351390779018,
+      "rewards/accuracy_reward": 0.15178571664728224,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.2261904776096344,
+      "rewards/tag_count_reward": 0.4159226268529892,
       "step": 3
     },
     {
-      "completion_length": 2929.4702758789062,
+      "completion_length": 3326.1488647460938,
       "epoch": 0.11188811188811189,
-      "grad_norm": 0.23280778527259827,
-      "kl": 0.00044918060302734375,
+      "grad_norm": 0.9779719114303589,
+      "kl": 0.01001739501953125,
       "learning_rate": 1.5000000000000002e-05,
-      "loss": 0.0,
-      "reward": 0.3162202350795269,
-      "reward_std": 0.24618623778223991,
-      "rewards/accuracy_reward": 0.09523809514939785,
+      "loss": 0.0004,
+      "reward": 0.5312500074505806,
+      "reward_std": 0.18081304244697094,
+      "rewards/accuracy_reward": 0.10416666511446238,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.2209821492433548,
+      "rewards/tag_count_reward": 0.4270833507180214,
       "step": 4
     },
     {
-      "completion_length": 3607.6875610351562,
+      "completion_length": 3557.666748046875,
       "epoch": 0.13986013986013987,
-      "grad_norm": 0.18857119977474213,
-      "kl": 0.00334930419921875,
+      "grad_norm": 0.7781193256378174,
+      "kl": 0.0257568359375,
       "learning_rate": 2e-05,
-      "loss": 0.0001,
-      "reward": 0.3065476231276989,
-      "reward_std": 0.2614261731505394,
-      "rewards/accuracy_reward": 0.14583333395421505,
+      "loss": 0.001,
+      "reward": 0.5401785746216774,
+      "reward_std": 0.17106053419411182,
+      "rewards/accuracy_reward": 0.1547619067132473,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.16071428544819355,
+      "rewards/tag_count_reward": 0.385416679084301,
       "step": 5
     },
     {
-      "completion_length": 4005.02685546875,
+      "completion_length": 3472.3839111328125,
       "epoch": 0.16783216783216784,
-      "grad_norm": 0.4723397195339203,
-      "kl": 0.050689697265625,
+      "grad_norm": 0.3135736286640167,
+      "kl": 0.0687255859375,
       "learning_rate": 1.995184726672197e-05,
-      "loss": 0.002,
-      "reward": 0.19345238618552685,
-      "reward_std": 0.22516153007745743,
-      "rewards/accuracy_reward": 0.06845238339155912,
+      "loss": 0.0027,
+      "reward": 0.5096726268529892,
+      "reward_std": 0.1920771636068821,
+      "rewards/accuracy_reward": 0.10714285913854837,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.12500000558793545,
+      "rewards/tag_count_reward": 0.4025297686457634,
       "step": 6
     },
     {
-      "completion_length": 4122.452392578125,
+      "completion_length": 3685.6577758789062,
       "epoch": 0.1958041958041958,
-      "grad_norm": 0.1486005038022995,
-      "kl": 0.0227203369140625,
+      "grad_norm": 0.23582090437412262,
+      "kl": 0.0960693359375,
       "learning_rate": 1.9807852804032306e-05,
-      "loss": 0.0009,
-      "reward": 0.17633929196745157,
-      "reward_std": 0.20393076166510582,
-      "rewards/accuracy_reward": 0.07142857275903225,
+      "loss": 0.0038,
+      "reward": 0.5394345298409462,
+      "reward_std": 0.1671802718192339,
+      "rewards/accuracy_reward": 0.14583333395421505,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.10491071548312902,
+      "rewards/tag_count_reward": 0.3936012014746666,
       "step": 7
     },
     {
-      "completion_length": 3902.4583129882812,
+      "completion_length": 3511.9107666015625,
       "epoch": 0.22377622377622378,
-      "grad_norm": 0.11479471623897552,
-      "kl": 0.019866943359375,
+      "grad_norm": 0.19193226099014282,
+      "kl": 0.146484375,
       "learning_rate": 1.956940335732209e-05,
-      "loss": 0.0008,
-      "reward": 0.17261905409395695,
-      "reward_std": 0.1820010170340538,
-      "rewards/accuracy_reward": 0.04761904873885214,
+      "loss": 0.0059,
+      "reward": 0.5066964402794838,
+      "reward_std": 0.1798535594716668,
+      "rewards/accuracy_reward": 0.11309523973613977,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.12500000558793545,
+      "rewards/tag_count_reward": 0.3936012014746666,
       "step": 8
     },
     {
-      "completion_length": 3809.5000610351562,
+      "completion_length": 3710.8781127929688,
       "epoch": 0.2517482517482518,
-      "grad_norm": 0.13024798035621643,
-      "kl": 0.02362060546875,
+      "grad_norm": 0.15887923538684845,
+      "kl": 0.1806640625,
       "learning_rate": 1.9238795325112867e-05,
-      "loss": 0.0009,
-      "reward": 0.2455357201397419,
-      "reward_std": 0.24880634248256683,
-      "rewards/accuracy_reward": 0.10714286006987095,
+      "loss": 0.0072,
+      "reward": 0.543898805975914,
+      "reward_std": 0.16826315969228745,
+      "rewards/accuracy_reward": 0.1636904780752957,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.1383928582072258,
+      "rewards/tag_count_reward": 0.3802083432674408,
       "step": 9
     },
     {
-      "completion_length": 3733.8929443359375,
+      "completion_length": 3805.0596313476562,
       "epoch": 0.27972027972027974,
-      "grad_norm": 0.13345155119895935,
-      "kl": 0.029022216796875,
+      "grad_norm": 0.17851299047470093,
+      "kl": 0.225341796875,
       "learning_rate": 1.881921264348355e-05,
-      "loss": 0.0012,
-      "reward": 0.2239583432674408,
-      "reward_std": 0.23797475174069405,
-      "rewards/accuracy_reward": 0.08333333674818277,
+      "loss": 0.009,
+      "reward": 0.5074404925107956,
+      "reward_std": 0.1729382909834385,
+      "rewards/accuracy_reward": 0.13392857275903225,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.140625,
+      "rewards/tag_count_reward": 0.3735119104385376,
       "step": 10
     },
     {
-      "completion_length": 3744.1756591796875,
+      "completion_length": 3785.4822998046875,
       "epoch": 0.3076923076923077,
-      "grad_norm": 0.26249733567237854,
-      "kl": 0.045654296875,
+      "grad_norm": 0.18465878069400787,
+      "kl": 0.266845703125,
       "learning_rate": 1.8314696123025456e-05,
-      "loss": 0.0018,
-      "reward": 0.2172619104385376,
-      "reward_std": 0.2155022956430912,
-      "rewards/accuracy_reward": 0.056547621730715036,
+      "loss": 0.0107,
+      "reward": 0.4315476343035698,
+      "reward_std": 0.11403452791273594,
+      "rewards/accuracy_reward": 0.07738095452077687,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.16071428917348385,
+      "rewards/tag_count_reward": 0.3541666716337204,
       "step": 11
     },
     {
-      "completion_length": 3854.7827758789062,
+      "completion_length": 3463.0625610351562,
       "epoch": 0.3356643356643357,
-      "grad_norm": 0.19111429154872894,
-      "kl": 0.05322265625,
+      "grad_norm": 0.8323477506637573,
+      "kl": 0.28662109375,
       "learning_rate": 1.773010453362737e-05,
-      "loss": 0.0021,
-      "reward": 0.1696428582072258,
-      "reward_std": 0.21632104367017746,
-      "rewards/accuracy_reward": 0.029761905316263437,
+      "loss": 0.0115,
+      "reward": 0.4404762014746666,
+      "reward_std": 0.14237426407635212,
+      "rewards/accuracy_reward": 0.059523810632526875,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.13988095708191395,
+      "rewards/tag_count_reward": 0.380952388048172,
       "step": 12
     },
     {
-      "completion_length": 3603.0952758789062,
+      "completion_length": 3813.7142944335938,
       "epoch": 0.36363636363636365,
-      "grad_norm": 0.20008493959903717,
-      "kl": 0.04888916015625,
+      "grad_norm": 0.19555552303791046,
+      "kl": 0.30908203125,
       "learning_rate": 1.7071067811865477e-05,
-      "loss": 0.002,
-      "reward": 0.396577388048172,
-      "reward_std": 0.330210629850626,
-      "rewards/accuracy_reward": 0.2321428619325161,
+      "loss": 0.0124,
+      "reward": 0.6488095223903656,
+      "reward_std": 0.16268891375511885,
+      "rewards/accuracy_reward": 0.2946428656578064,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.1644345298409462,
+      "rewards/tag_count_reward": 0.354166679084301,
       "step": 13
     },
     {
-      "completion_length": 3576.5714721679688,
+      "completion_length": 3913.9733276367188,
       "epoch": 0.3916083916083916,
-      "grad_norm": 0.28434476256370544,
-      "kl": 0.07000732421875,
+      "grad_norm": 0.1619066596031189,
+      "kl": 0.3642578125,
       "learning_rate": 1.6343932841636455e-05,
-      "loss": 0.0028,
-      "reward": 0.4546131044626236,
-      "reward_std": 0.38984502851963043,
-      "rewards/accuracy_reward": 0.2797619141638279,
+      "loss": 0.0146,
+      "reward": 0.685267873108387,
+      "reward_std": 0.22458449006080627,
+      "rewards/accuracy_reward": 0.3452380932867527,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.174851194024086,
+      "rewards/tag_count_reward": 0.3400297686457634,
       "step": 14
     },
     {
-      "completion_length": 3697.041748046875,
+      "completion_length": 4126.169677734375,
       "epoch": 0.4195804195804196,
-      "grad_norm": 0.1858980506658554,
-      "kl": 0.076171875,
+      "grad_norm": 183.4498291015625,
+      "kl": 1.22119140625,
       "learning_rate": 1.5555702330196024e-05,
-      "loss": 0.003,
-      "reward": 0.351190485060215,
-      "reward_std": 0.3252423033118248,
-      "rewards/accuracy_reward": 0.1875000037252903,
+      "loss": 0.0489,
+      "reward": 0.5327381044626236,
+      "reward_std": 0.168159706518054,
+      "rewards/accuracy_reward": 0.21726191230118275,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.1636904813349247,
+      "rewards/tag_count_reward": 0.3154762014746666,
       "step": 15
     },
     {
-      "completion_length": 3621.52685546875,
+      "completion_length": 4069.3423461914062,
       "epoch": 0.44755244755244755,
-      "grad_norm": 0.2509734034538269,
-      "kl": 0.0791015625,
+      "grad_norm": 0.14787977933883667,
+      "kl": 0.39599609375,
       "learning_rate": 1.4713967368259981e-05,
-      "loss": 0.0032,
-      "reward": 0.4129464328289032,
-      "reward_std": 0.36431775987148285,
-      "rewards/accuracy_reward": 0.2291666679084301,
+      "loss": 0.0158,
+      "reward": 0.5260416716337204,
+      "reward_std": 0.1734736319631338,
+      "rewards/accuracy_reward": 0.21130952797830105,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.1837797649204731,
+      "rewards/tag_count_reward": 0.3147321492433548,
       "step": 16
     },
     {
-      "completion_length": 3533.1578369140625,
+      "completion_length": 4006.3988647460938,
       "epoch": 0.4755244755244755,
-      "grad_norm": 0.20609919726848602,
-      "kl": 0.1068115234375,
+      "grad_norm": 7.54030704498291,
+      "kl": 0.4892578125,
       "learning_rate": 1.3826834323650899e-05,
-      "loss": 0.0043,
-      "reward": 0.376488097012043,
-      "reward_std": 0.36176759749650955,
-      "rewards/accuracy_reward": 0.20238095708191395,
+      "loss": 0.0196,
+      "reward": 0.5386904776096344,
+      "reward_std": 0.21569561585783958,
+      "rewards/accuracy_reward": 0.21726191090419888,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.1741071492433548,
+      "rewards/tag_count_reward": 0.321428582072258,
       "step": 17
     },
     {
-      "completion_length": 3520.2650146484375,
+      "completion_length": 3950.7174072265625,
       "epoch": 0.5034965034965035,
-      "grad_norm": 0.17171701788902283,
-      "kl": 0.09619140625,
+      "grad_norm": 0.24552476406097412,
+      "kl": 0.3994140625,
       "learning_rate": 1.2902846772544625e-05,
-      "loss": 0.0038,
-      "reward": 0.3824404925107956,
-      "reward_std": 0.35053466260433197,
-      "rewards/accuracy_reward": 0.1964285708963871,
+      "loss": 0.016,
+      "reward": 0.5223214477300644,
+      "reward_std": 0.21196102164685726,
+      "rewards/accuracy_reward": 0.19345238246023655,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.1860119104385376,
+      "rewards/tag_count_reward": 0.3288690596818924,
       "step": 18
     },
     {
-      "completion_length": 3493.7142944335938,
+      "completion_length": 3340.389892578125,
       "epoch": 0.5314685314685315,
-      "grad_norm": 0.2554529905319214,
-      "kl": 0.0972900390625,
+      "grad_norm": 0.2469428926706314,
+      "kl": 0.3818359375,
       "learning_rate": 1.1950903220161286e-05,
-      "loss": 0.0039,
-      "reward": 0.4367559552192688,
-      "reward_std": 0.4041043147444725,
-      "rewards/accuracy_reward": 0.2321428656578064,
+      "loss": 0.0153,
+      "reward": 0.7261904776096344,
+      "reward_std": 0.3062071055173874,
+      "rewards/accuracy_reward": 0.3363095298409462,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.2046131007373333,
+      "rewards/tag_count_reward": 0.3898809626698494,
       "step": 19
     },
     {
-      "completion_length": 3441.1636962890625,
+      "completion_length": 3832.21142578125,
       "epoch": 0.5594405594405595,
-      "grad_norm": 0.19453173875808716,
-      "kl": 0.1204833984375,
+      "grad_norm": 0.20858395099639893,
+      "kl": 0.4306640625,
       "learning_rate": 1.098017140329561e-05,
-      "loss": 0.0048,
-      "reward": 0.387648805975914,
-      "reward_std": 0.3576350286602974,
-      "rewards/accuracy_reward": 0.1815476231276989,
+      "loss": 0.0172,
+      "reward": 0.5424107238650322,
+      "reward_std": 0.277970090508461,
+      "rewards/accuracy_reward": 0.18452381156384945,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.206101194024086,
+      "rewards/tag_count_reward": 0.357886902987957,
       "step": 20
     },
     {
-      "completion_length": 3643.386962890625,
+      "completion_length": 3544.5535888671875,
       "epoch": 0.5874125874125874,
-      "grad_norm": 0.20605766773223877,
-      "kl": 0.150390625,
+      "grad_norm": 0.8287240266799927,
+      "kl": 0.39404296875,
       "learning_rate": 1e-05,
-      "loss": 0.006,
-      "reward": 0.433779776096344,
-      "reward_std": 0.4044775441288948,
-      "rewards/accuracy_reward": 0.2351190522313118,
+      "loss": 0.0158,
+      "reward": 0.5654762089252472,
+      "reward_std": 0.29340360686182976,
+      "rewards/accuracy_reward": 0.19642857648432255,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.1986607201397419,
+      "rewards/tag_count_reward": 0.3690476268529892,
       "step": 21
     },
     {
-      "completion_length": 3483.3482666015625,
+      "completion_length": 3684.431640625,
       "epoch": 0.6153846153846154,
-      "grad_norm": 0.26148831844329834,
-      "kl": 0.164794921875,
+      "grad_norm": 0.18662597239017487,
+      "kl": 0.3759765625,
       "learning_rate": 9.019828596704394e-06,
-      "loss": 0.0066,
-      "reward": 0.432291679084301,
-      "reward_std": 0.40663766860961914,
-      "rewards/accuracy_reward": 0.20535714365541935,
+      "loss": 0.015,
+      "reward": 0.5915178656578064,
+      "reward_std": 0.26311009749770164,
+      "rewards/accuracy_reward": 0.22619047947227955,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.2269345298409462,
+      "rewards/tag_count_reward": 0.365327388048172,
       "step": 22
     },
     {
-      "completion_length": 3216.8928833007812,
+      "completion_length": 3296.729248046875,
       "epoch": 0.6433566433566433,
-      "grad_norm": 0.9234132766723633,
-      "kl": 0.19677734375,
+      "grad_norm": 2.3806121349334717,
+      "kl": 0.42236328125,
       "learning_rate": 8.04909677983872e-06,
-      "loss": 0.0079,
-      "reward": 0.5305059626698494,
-      "reward_std": 0.4592476785182953,
-      "rewards/accuracy_reward": 0.2857142835855484,
+      "loss": 0.0169,
+      "reward": 0.7991071492433548,
+      "reward_std": 0.3090755343437195,
+      "rewards/accuracy_reward": 0.3988095298409462,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.2447916716337204,
+      "rewards/tag_count_reward": 0.4002976268529892,
       "step": 23
     },
     {
-      "completion_length": 3309.6517944335938,
+      "completion_length": 3165.3363647460938,
       "epoch": 0.6713286713286714,
-      "grad_norm": 0.43054357171058655,
-      "kl": 0.211669921875,
+      "grad_norm": 0.24191996455192566,
+      "kl": 0.3544921875,
       "learning_rate": 7.097153227455379e-06,
-      "loss": 0.0085,
-      "reward": 0.4821428805589676,
-      "reward_std": 0.421218641102314,
-      "rewards/accuracy_reward": 0.23511905036866665,
+      "loss": 0.0142,
+      "reward": 0.769345261156559,
+      "reward_std": 0.3235185295343399,
+      "rewards/accuracy_reward": 0.342261902987957,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.2470238134264946,
+      "rewards/tag_count_reward": 0.4270833358168602,
       "step": 24
     },
     {
-      "completion_length": 3227.5625610351562,
+      "completion_length": 3155.2977294921875,
       "epoch": 0.6993006993006993,
-      "grad_norm": 0.47960782051086426,
-      "kl": 0.154541015625,
+      "grad_norm": 0.2336803525686264,
+      "kl": 0.34521484375,
       "learning_rate": 6.173165676349103e-06,
-      "loss": 0.0062,
-      "reward": 0.5811012014746666,
-      "reward_std": 0.45310985296964645,
-      "rewards/accuracy_reward": 0.3005952425301075,
+      "loss": 0.0138,
+      "reward": 0.8110119253396988,
+      "reward_std": 0.30220678821206093,
+      "rewards/accuracy_reward": 0.3630952462553978,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.2805059626698494,
+      "rewards/tag_count_reward": 0.4479166716337204,
       "step": 25
     },
     {
-      "completion_length": 3407.9524536132812,
+      "completion_length": 3391.669677734375,
       "epoch": 0.7272727272727273,
-      "grad_norm": 0.22348760068416595,
-      "kl": 0.145263671875,
+      "grad_norm": 0.20384787023067474,
+      "kl": 0.38623046875,
       "learning_rate": 5.286032631740023e-06,
-      "loss": 0.0058,
-      "reward": 0.5096726343035698,
-      "reward_std": 0.36870553344488144,
-      "rewards/accuracy_reward": 0.2261904776096344,
+      "loss": 0.0154,
+      "reward": 0.7068452686071396,
+      "reward_std": 0.22004729695618153,
+      "rewards/accuracy_reward": 0.318452388048172,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.2834821417927742,
+      "rewards/tag_count_reward": 0.3883928656578064,
       "step": 26
     },
     {
-      "completion_length": 3143.4613647460938,
+      "completion_length": 2888.4583740234375,
       "epoch": 0.7552447552447552,
-      "grad_norm": 0.35857293009757996,
-      "kl": 0.1617431640625,
+      "grad_norm": 0.21734486520290375,
+      "kl": 0.36328125,
       "learning_rate": 4.444297669803981e-06,
-      "loss": 0.0065,
-      "reward": 0.5260416865348816,
-      "reward_std": 0.38500337302684784,
-      "rewards/accuracy_reward": 0.23214286379516125,
+      "loss": 0.0145,
+      "reward": 0.8251488208770752,
+      "reward_std": 0.23692942410707474,
+      "rewards/accuracy_reward": 0.354166679084301,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.293898805975914,
+      "rewards/tag_count_reward": 0.4709821566939354,
       "step": 27
     },
     {
-      "completion_length": 3477.8214721679688,
+      "completion_length": 3569.511962890625,
       "epoch": 0.7832167832167832,
-      "grad_norm": 0.22233816981315613,
-      "kl": 0.16357421875,
+      "grad_norm": 0.22490565478801727,
+      "kl": 0.38720703125,
       "learning_rate": 3.6560671583635467e-06,
-      "loss": 0.0065,
-      "reward": 0.3973214253783226,
-      "reward_std": 0.297023706138134,
-      "rewards/accuracy_reward": 0.11309523927047849,
+      "loss": 0.0155,
+      "reward": 0.5215774029493332,
+      "reward_std": 0.1440001018345356,
+      "rewards/accuracy_reward": 0.12797619495540857,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.284226194024086,
+      "rewards/tag_count_reward": 0.3936012014746666,
       "step": 28
     },
     {
-      "completion_length": 3345.33935546875,
+      "completion_length": 3503.2530517578125,
       "epoch": 0.8111888111888111,
-      "grad_norm": 0.3590240180492401,
-      "kl": 0.154541015625,
+      "grad_norm": 0.161085844039917,
+      "kl": 0.32861328125,
       "learning_rate": 2.9289321881345257e-06,
-      "loss": 0.0062,
-      "reward": 0.5654762014746666,
-      "reward_std": 0.359061636030674,
-      "rewards/accuracy_reward": 0.25892857648432255,
+      "loss": 0.0131,
+      "reward": 0.658482164144516,
+      "reward_std": 0.23359560035169125,
+      "rewards/accuracy_reward": 0.23809524439275265,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.3065476268529892,
+      "rewards/tag_count_reward": 0.4203869178891182,
       "step": 29
     },
     {
-      "completion_length": 3411.324462890625,
+      "completion_length": 3520.7709350585938,
       "epoch": 0.8391608391608392,
-      "grad_norm": 0.2620408833026886,
-      "kl": 0.161376953125,
+      "grad_norm": 0.6872840523719788,
+      "kl": 0.3876953125,
       "learning_rate": 2.26989546637263e-06,
-      "loss": 0.0065,
-      "reward": 0.4680059552192688,
-      "reward_std": 0.3296178914606571,
-      "rewards/accuracy_reward": 0.15773810190148652,
+      "loss": 0.0155,
+      "reward": 0.5461309626698494,
+      "reward_std": 0.1894807331264019,
+      "rewards/accuracy_reward": 0.14880952727980912,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.3102678656578064,
+      "rewards/tag_count_reward": 0.3973214328289032,
       "step": 30
     },
     {
-      "completion_length": 3285.4940185546875,
+      "completion_length": 3472.7084350585938,
       "epoch": 0.8671328671328671,
-      "grad_norm": 0.27508166432380676,
-      "kl": 0.1549072265625,
+      "grad_norm": 0.19596825540065765,
+      "kl": 0.345703125,
       "learning_rate": 1.6853038769745466e-06,
-      "loss": 0.0062,
-      "reward": 0.5357143133878708,
-      "reward_std": 0.3590538948774338,
-      "rewards/accuracy_reward": 0.2351190522313118,
+      "loss": 0.0138,
+      "reward": 0.6770833432674408,
+      "reward_std": 0.2258823774755001,
+      "rewards/accuracy_reward": 0.2589285783469677,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.3005952388048172,
+      "rewards/tag_count_reward": 0.4181547686457634,
       "step": 31
     },
     {
-      "completion_length": 3199.0447387695312,
+      "completion_length": 3174.8154907226562,
       "epoch": 0.8951048951048951,
-      "grad_norm": 6.951571941375732,
-      "kl": 0.44775390625,
+      "grad_norm": 0.16952919960021973,
+      "kl": 0.32080078125,
       "learning_rate": 1.1807873565164507e-06,
-      "loss": 0.0179,
-      "reward": 0.5491071566939354,
-      "reward_std": 0.34805624932050705,
-      "rewards/accuracy_reward": 0.2380952425301075,
+      "loss": 0.0128,
+      "reward": 0.7038690596818924,
+      "reward_std": 0.2037985846400261,
+      "rewards/accuracy_reward": 0.2619047686457634,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.3110119178891182,
+      "rewards/tag_count_reward": 0.4419642984867096,
       "step": 32
     },
     {
-      "completion_length": 3249.9732666015625,
+      "completion_length": 3343.431640625,
       "epoch": 0.9230769230769231,
-      "grad_norm": 0.3479785621166229,
-      "kl": 0.165283203125,
+      "grad_norm": 0.38616687059402466,
+      "kl": 0.380859375,
       "learning_rate": 7.612046748871327e-07,
-      "loss": 0.0066,
-      "reward": 0.5736607313156128,
-      "reward_std": 0.3619985207915306,
-      "rewards/accuracy_reward": 0.2440476231276989,
+      "loss": 0.0152,
+      "reward": 0.6845238357782364,
+      "reward_std": 0.25144897773861885,
+      "rewards/accuracy_reward": 0.258928582072258,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.3296131044626236,
+      "rewards/tag_count_reward": 0.4255952462553978,
       "step": 33
     },
     {
-      "completion_length": 3028.9494018554688,
+      "completion_length": 2863.6845703125,
       "epoch": 0.951048951048951,
-      "grad_norm": 0.3831622302532196,
-      "kl": 0.178466796875,
+      "grad_norm": 55.34281539916992,
+      "kl": 1.37109375,
       "learning_rate": 4.305966426779118e-07,
-      "loss": 0.0071,
-      "reward": 0.5811012089252472,
-      "reward_std": 0.37812189757823944,
-      "rewards/accuracy_reward": 0.2678571492433548,
+      "loss": 0.0549,
+      "reward": 0.8660714328289032,
+      "reward_std": 0.29025010019540787,
+      "rewards/accuracy_reward": 0.348214291036129,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.3132440596818924,
+      "rewards/tag_count_reward": 0.5178571417927742,
       "step": 34
     },
     {
-      "completion_length": 3763.1279907226562,
+      "completion_length": 3796.6666870117188,
       "epoch": 0.9790209790209791,
-      "grad_norm": 0.1765451729297638,
-      "kl": 0.155029296875,
+      "grad_norm": 0.18026739358901978,
+      "kl": 0.412109375,
       "learning_rate": 1.921471959676957e-07,
-      "loss": 0.0062,
-      "reward": 0.3869047686457634,
-      "reward_std": 0.28744565322995186,
-      "rewards/accuracy_reward": 0.13095238525420427,
+      "loss": 0.0165,
+      "reward": 0.4620535895228386,
+      "reward_std": 0.18349270708858967,
+      "rewards/accuracy_reward": 0.09523809794336557,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.2559523917734623,
+      "rewards/tag_count_reward": 0.366815485060215,
       "step": 35
     },
     {
-      "completion_length": 3817.74560546875,
+      "completion_length": 4218.495524088542,
       "epoch": 1.0,
-      "grad_norm": 0.1765451729297638,
-      "kl": 0.17024739583333334,
+      "grad_norm": 0.18026739358901978,
+      "kl": 0.4368489583333333,
       "learning_rate": 4.815273327803183e-08,
-      "loss": 0.0051,
-      "reward": 0.3432539800802867,
-      "reward_std": 0.2780213952064514,
-      "rewards/accuracy_reward": 0.0912698432803154,
+      "loss": 0.0131,
+      "reward": 0.3720238208770752,
+      "reward_std": 0.15442068378130594,
+      "rewards/accuracy_reward": 0.0476190485060215,
       "rewards/format_reward": 0.0,
-      "rewards/tag_count_reward": 0.2519841293493907,
+      "rewards/tag_count_reward": 0.3244047661622365,
       "step": 36
     },
     {
       "epoch": 1.0,
       "step": 36,
       "total_flos": 0.0,
-      "train_loss": 0.004263622482312219,
-      "train_runtime": 9387.6074,
-      "train_samples_per_second": 0.107,
-      "train_steps_per_second": 0.004
+      "train_loss": 0.013564518235874741,
+      "train_runtime": 16822.299,
+      "train_samples_per_second": 0.059,
+      "train_steps_per_second": 0.002
     }
   ],
   "logging_steps": 1,