End of training

README.md

@@ -1,20 +1,20 @@
 ---
-base_model: Gensyn/Qwen2.5-0.5B-Instruct
+base_model: unsloth/Qwen2.5-0.5B-Instruct
 library_name: transformers
 model_name: Qwen2.5-0.5B-Instruct-Gensyn-Swarm-quick_timid_frog
 tags:
 - generated_from_trainer
-- rl-swarm
 - grpo
 - gensyn
 - I am quick timid frog
 - trl
+- rl-swarm
 licence: license
 ---
 
 # Model Card for Qwen2.5-0.5B-Instruct-Gensyn-Swarm-quick_timid_frog
 
-This model is a fine-tuned version of [Gensyn/Qwen2.5-0.5B-Instruct](https://huggingface.co/Gensyn/Qwen2.5-0.5B-Instruct).
+This model is a fine-tuned version of [unsloth/Qwen2.5-0.5B-Instruct](https://huggingface.co/unsloth/Qwen2.5-0.5B-Instruct).
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start
@@ -37,10 +37,10 @@ This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing
 
 ### Framework versions
 
-- TRL: 0.15.2
-- Transformers: 4.51.2
-- Pytorch: 2.5.1
-- Datasets: 3.5.0
+- TRL: 0.19.0
+- Transformers: 4.52.4
+- Pytorch: 2.7.1
+- Datasets: 3.6.0
 - Tokenizers: 0.21.1
 
 ## Citations
@@ -62,7 +62,7 @@ Cite TRL as:
 ```bibtex
 @misc{vonwerra2022trl,
 	title        = {{TRL: Transformer Reinforcement Learning}},
-	author       = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallouédec},
+	author       = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallou{\'e}dec},
 	year         = 2020,
 	journal      = {GitHub repository},
 	publisher    = {GitHub},

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.12956260753126117,
-    "train_runtime": 123.8534,
-    "train_samples": 28,
-    "train_samples_per_second": 2.584,
-    "train_steps_per_second": 0.161
+    "train_loss": 0.03613492660224438,
+    "train_runtime": 1374.3618,
+    "train_samples": 83,
+    "train_samples_per_second": 0.116,
+    "train_steps_per_second": 0.015
 }

chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

config.json

@@ -3,7 +3,6 @@
     "Qwen2ForCausalLM"
   ],
   "attention_dropout": 0.0,
-  "bos_token_id": 151643,
   "eos_token_id": 151645,
   "hidden_act": "silu",
   "hidden_size": 896,
@@ -15,13 +14,15 @@
   "num_attention_heads": 14,
   "num_hidden_layers": 24,
   "num_key_value_heads": 2,
+  "pad_token_id": 151654,
   "rms_norm_eps": 1e-06,
   "rope_scaling": null,
   "rope_theta": 1000000.0,
-  "sliding_window": 32768,
+  "sliding_window": null,
   "tie_word_embeddings": true,
   "torch_dtype": "float32",
-  "transformers_version": "4.51.2",
+  "transformers_version": "4.52.4",
+  "unsloth_fixed": true,
   "use_cache": true,
   "use_sliding_window": false,
   "vocab_size": 151936

generation_config.json

@@ -5,10 +5,11 @@
     151645,
     151643
   ],
-  "pad_token_id": 151643,
+  "max_length": 32768,
+  "pad_token_id": 151654,
   "repetition_penalty": 1.1,
   "temperature": 0.7,
   "top_k": 20,
   "top_p": 0.8,
-  "transformers_version": "4.51.2"
+  "transformers_version": "4.52.4"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:19b9b6f34a86f4c7236d477f7ba60b1c001b120b58c0abea0fa85431e70e8c5c
+oid sha256:866a08c94c86a17ef6575016471b9aaf5cca32cdb45de267c72663a76249e8ca
 size 1976163472

special_tokens_map.json

@@ -22,7 +22,7 @@
     "single_word": false
   },
   "pad_token": {
-    "content": "<|endoftext|>",
+    "content": "<|vision_pad|>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5eee858c5123a4279c3e1f7b81247343f356ac767940b2692a928ad929543214
-size 11422063
+oid sha256:64e71213db910f5cafa86d35091f37393dcc344b1bbc34091d1b3eed4cca01d5
+size 11422064

tokenizer_config.json

@@ -195,13 +195,13 @@
     "<|video_pad|>"
   ],
   "bos_token": null,
-  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|im_end|>",
   "errors": "replace",
   "extra_special_tokens": {},
-  "model_max_length": 131072,
-  "pad_token": "<|endoftext|>",
+  "model_max_length": 32768,
+  "pad_token": "<|vision_pad|>",
+  "padding_side": "left",
   "split_special_tokens": false,
   "tokenizer_class": "Qwen2Tokenizer",
   "unk_token": null

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.12956260753126117,
-    "train_runtime": 123.8534,
-    "train_samples": 28,
-    "train_samples_per_second": 2.584,
-    "train_steps_per_second": 0.161
+    "train_loss": 0.03613492660224438,
+    "train_runtime": 1374.3618,
+    "train_samples": 83,
+    "train_samples_per_second": 0.116,
+    "train_steps_per_second": 0.015
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 5.0,
+  "epoch": 0.963855421686747,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,209 +10,419 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 249.875,
-      "epoch": 0.5714285714285714,
-      "grad_norm": 47.85331726074219,
-      "kl": 0.0,
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0625,
+      "completions/max_length": 768.5,
+      "completions/max_terminated_length": 544.0,
+      "completions/mean_length": 282.8125,
+      "completions/mean_terminated_length": 233.37500762939453,
+      "completions/min_length": 35.5,
+      "completions/min_terminated_length": 35.5,
+      "epoch": 0.0963855421686747,
+      "frac_reward_zero_std": 0.125,
+      "grad_norm": 113.29230499267578,
+      "kl": -6.7503988510075885e-09,
       "learning_rate": 5e-07,
-      "loss": 0.0,
-      "reward": 3.0323707722127438,
-      "reward_std": 0.8131099180318415,
-      "rewards/concensus_correctness_reward_func": 0.6197499856352806,
-      "rewards/consensus_reward_func": 0.875,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6093395496718585,
-      "rewards/soft_format_reward_func": 0.015625,
-      "rewards/strict_format_reward_func": 0.15625,
-      "rewards/xmlcount_reward_func": 0.7564062550663948,
+      "loss": -0.0304,
+      "num_tokens": 8621.0,
+      "reward": 0.05961298104375601,
+      "reward_std": 0.021443639416247606,
+      "rewards/concensus_correctness_reward_func/mean": 0.0,
+      "rewards/concensus_correctness_reward_func/std": 0.0,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
+      "rewards/cumulative_reward_2/mean": 0.0,
+      "rewards/cumulative_reward_2/std": 0.0,
+      "rewards/final_correctness_reward_func/mean": 0.0,
+      "rewards/final_correctness_reward_func/std": 0.0,
+      "rewards/question_recreation_reward_func/mean": 0.05961298104375601,
+      "rewards/question_recreation_reward_func/std": 0.03218379570171237,
+      "rewards/soft_format_reward_func/mean": 0.0,
+      "rewards/soft_format_reward_func/std": 0.0,
+      "rewards/strict_format_reward_func/mean": 0.0,
+      "rewards/strict_format_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": 0.0,
+      "rewards/xmlcount_reward_func/std": 0.0,
       "step": 2
     },
     {
-      "completion_length": 149.91666666666666,
-      "epoch": 1.0,
-      "grad_norm": 103.94192504882812,
-      "kl": 0.13347215698255846,
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 881.0,
+      "completions/max_terminated_length": 881.0,
+      "completions/mean_length": 393.1875,
+      "completions/mean_terminated_length": 393.1875,
+      "completions/min_length": 62.0,
+      "completions/min_terminated_length": 62.0,
+      "epoch": 0.1927710843373494,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 6.477648735046387,
+      "kl": 0.00010615352056220217,
       "learning_rate": 4.864543104251586e-07,
-      "loss": 0.0001,
-      "reward": 5.739850004514058,
-      "reward_std": 0.6970982489486536,
-      "rewards/concensus_correctness_reward_func": 1.7566666553417842,
-      "rewards/consensus_reward_func": 1.5833333333333333,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.8079750432322422,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3333333333333333,
-      "rewards/xmlcount_reward_func": 1.175208330154419,
+      "loss": -0.0486,
+      "num_tokens": 19008.0,
+      "reward": 0.018797683529555798,
+      "reward_std": 0.007865483523346484,
+      "rewards/concensus_correctness_reward_func/mean": 0.0,
+      "rewards/concensus_correctness_reward_func/std": 0.0,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
+      "rewards/cumulative_reward_2/mean": 0.0,
+      "rewards/cumulative_reward_2/std": 0.0,
+      "rewards/final_correctness_reward_func/mean": 0.0,
+      "rewards/final_correctness_reward_func/std": 0.0,
+      "rewards/question_recreation_reward_func/mean": 0.018797683529555798,
+      "rewards/question_recreation_reward_func/std": 0.00972810504026711,
+      "rewards/soft_format_reward_func/mean": 0.0,
+      "rewards/soft_format_reward_func/std": 0.0,
+      "rewards/strict_format_reward_func/mean": 0.0,
+      "rewards/strict_format_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": 0.0,
+      "rewards/xmlcount_reward_func/std": 0.0,
       "step": 4
     },
     {
-      "completion_length": 158.9375,
-      "epoch": 1.5714285714285714,
-      "grad_norm": 89.15827941894531,
-      "kl": 1.9806956076063216,
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 806.5,
+      "completions/max_terminated_length": 806.5,
+      "completions/mean_length": 288.1875,
+      "completions/mean_terminated_length": 288.1875,
+      "completions/min_length": 50.0,
+      "completions/min_terminated_length": 50.0,
+      "epoch": 0.2891566265060241,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 7.433625221252441,
+      "kl": 0.0004048098953717272,
       "learning_rate": 4.472851273490984e-07,
-      "loss": 0.002,
-      "reward": 5.2741532772779465,
-      "reward_std": 0.8553773319144966,
-      "rewards/concensus_correctness_reward_func": 1.6266249865293503,
-      "rewards/consensus_reward_func": 1.5,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.7314033512957394,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.234375,
-      "rewards/xmlcount_reward_func": 1.1192499995231628,
+      "loss": 0.0337,
+      "num_tokens": 27715.0,
+      "reward": 0.08134639449417591,
+      "reward_std": 0.033843206241726875,
+      "rewards/concensus_correctness_reward_func/mean": 0.0,
+      "rewards/concensus_correctness_reward_func/std": 0.0,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
+      "rewards/cumulative_reward_2/mean": 0.0,
+      "rewards/cumulative_reward_2/std": 0.0,
+      "rewards/final_correctness_reward_func/mean": 0.0,
+      "rewards/final_correctness_reward_func/std": 0.0,
+      "rewards/question_recreation_reward_func/mean": 0.09159639663994312,
+      "rewards/question_recreation_reward_func/std": 0.03545556031167507,
+      "rewards/soft_format_reward_func/mean": 0.0,
+      "rewards/soft_format_reward_func/std": 0.0,
+      "rewards/strict_format_reward_func/mean": 0.0,
+      "rewards/strict_format_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": -0.010250000283122063,
+      "rewards/xmlcount_reward_func/std": 0.028991378843784332,
       "step": 6
     },
     {
-      "completion_length": 211.54166666666666,
-      "epoch": 2.0,
-      "grad_norm": 115.98817443847656,
-      "kl": 8.089183079699675,
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 443.0,
+      "completions/max_terminated_length": 443.0,
+      "completions/mean_length": 268.3125,
+      "completions/mean_terminated_length": 268.3125,
+      "completions/min_length": 39.0,
+      "completions/min_terminated_length": 39.0,
+      "epoch": 0.3855421686746988,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 5.330082416534424,
+      "kl": 0.0007420043511956464,
       "learning_rate": 3.867370395306068e-07,
-      "loss": 0.0061,
-      "reward": 4.624983638525009,
-      "reward_std": 0.8529471913352609,
-      "rewards/concensus_correctness_reward_func": 1.2500833123922348,
-      "rewards/consensus_reward_func": 1.5,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6848169888059298,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.14583333333333334,
-      "rewards/xmlcount_reward_func": 1.0442500114440918,
+      "loss": 0.0677,
+      "num_tokens": 36104.0,
+      "reward": 0.13727860897779465,
+      "reward_std": 0.04922554735094309,
+      "rewards/concensus_correctness_reward_func/mean": 0.0,
+      "rewards/concensus_correctness_reward_func/std": 0.0,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
+      "rewards/cumulative_reward_2/mean": 0.0,
+      "rewards/cumulative_reward_2/std": 0.0,
+      "rewards/final_correctness_reward_func/mean": 0.0,
+      "rewards/final_correctness_reward_func/std": 0.0,
+      "rewards/question_recreation_reward_func/mean": 0.12946611270308495,
+      "rewards/question_recreation_reward_func/std": 0.10384266264736652,
+      "rewards/soft_format_reward_func/mean": 0.0,
+      "rewards/soft_format_reward_func/std": 0.0,
+      "rewards/strict_format_reward_func/mean": 0.0,
+      "rewards/strict_format_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": 0.0078125,
+      "rewards/xmlcount_reward_func/std": 0.022097086533904076,
       "step": 8
     },
     {
-      "completion_length": 158.53125,
-      "epoch": 2.571428571428571,
-      "grad_norm": 128.32345581054688,
-      "kl": 10.754117728210986,
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1875,
+      "completions/max_length": 1024.0,
+      "completions/max_terminated_length": 889.5,
+      "completions/mean_length": 506.625,
+      "completions/mean_terminated_length": 393.5833435058594,
+      "completions/min_length": 29.5,
+      "completions/min_terminated_length": 29.5,
+      "epoch": 0.4819277108433735,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 5.515347957611084,
+      "kl": 0.00031582830342813395,
       "learning_rate": 3.1137137178519977e-07,
-      "loss": 0.0108,
-      "reward": 4.959709584712982,
-      "reward_std": 1.3167340854997747,
-      "rewards/concensus_correctness_reward_func": 1.4026249796152115,
-      "rewards/consensus_reward_func": 1.3125,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.7500846465118229,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.296875,
-      "rewards/xmlcount_reward_func": 1.135124996304512,
+      "loss": 0.0551,
+      "num_tokens": 48306.0,
+      "reward": 0.04514514096081257,
+      "reward_std": 0.04106513550505042,
+      "rewards/concensus_correctness_reward_func/mean": 0.0,
+      "rewards/concensus_correctness_reward_func/std": 0.0,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
+      "rewards/cumulative_reward_2/mean": 0.0,
+      "rewards/cumulative_reward_2/std": 0.0,
+      "rewards/final_correctness_reward_func/mean": 0.0,
+      "rewards/final_correctness_reward_func/std": 0.0,
+      "rewards/question_recreation_reward_func/mean": 0.019582641310989857,
+      "rewards/question_recreation_reward_func/std": 0.013477418571710587,
+      "rewards/soft_format_reward_func/mean": 0.0,
+      "rewards/soft_format_reward_func/std": 0.0,
+      "rewards/strict_format_reward_func/mean": 0.0,
+      "rewards/strict_format_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": 0.025562500581145287,
+      "rewards/xmlcount_reward_func/std": 0.07230167090892792,
       "step": 10
     },
     {
-      "completion_length": 184.33333333333334,
-      "epoch": 3.0,
-      "grad_norm": 24.66672706604004,
-      "kl": 3.1319144380589328,
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 685.0,
+      "completions/max_terminated_length": 685.0,
+      "completions/mean_length": 216.75,
+      "completions/mean_terminated_length": 216.75,
+      "completions/min_length": 35.5,
+      "completions/min_terminated_length": 35.5,
+      "epoch": 0.5783132530120482,
+      "frac_reward_zero_std": 0.125,
+      "grad_norm": 24.28751564025879,
+      "kl": 0.0019251212124800077,
       "learning_rate": 2.2935516363191693e-07,
-      "loss": 0.0023,
-      "reward": 5.048190355300903,
-      "reward_std": 0.7422210735579332,
-      "rewards/concensus_correctness_reward_func": 1.428333322207133,
-      "rewards/consensus_reward_func": 1.6666666666666667,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.7871903777122498,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.20833333333333334,
-      "rewards/xmlcount_reward_func": 0.9576666702826818,
+      "loss": 0.1978,
+      "num_tokens": 55870.0,
+      "reward": 0.1261079115793109,
+      "reward_std": 0.15609003114514053,
+      "rewards/concensus_correctness_reward_func/mean": 0.10837499797344208,
+      "rewards/concensus_correctness_reward_func/std": 0.30653080344200134,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
+      "rewards/cumulative_reward_2/mean": 0.0,
+      "rewards/cumulative_reward_2/std": 0.0,
+      "rewards/final_correctness_reward_func/mean": 0.0,
+      "rewards/final_correctness_reward_func/std": 0.0,
+      "rewards/question_recreation_reward_func/mean": 0.01773291453719139,
+      "rewards/question_recreation_reward_func/std": 0.008510306011885405,
+      "rewards/soft_format_reward_func/mean": 0.0,
+      "rewards/soft_format_reward_func/std": 0.0,
+      "rewards/strict_format_reward_func/mean": 0.0,
+      "rewards/strict_format_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": 0.0,
+      "rewards/xmlcount_reward_func/std": 0.0,
       "step": 12
     },
     {
-      "completion_length": 210.53125,
-      "epoch": 3.571428571428571,
-      "grad_norm": 440.6236267089844,
-      "kl": 37.00089144241065,
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.3125,
+      "completions/max_length": 1024.0,
+      "completions/max_terminated_length": 774.5,
+      "completions/mean_length": 514.625,
+      "completions/mean_terminated_length": 292.6999969482422,
+      "completions/min_length": 18.0,
+      "completions/min_terminated_length": 18.0,
+      "epoch": 0.6746987951807228,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 7.534204959869385,
+      "kl": 0.0008234772067226004,
       "learning_rate": 1.4957614383675767e-07,
-      "loss": 0.037,
-      "reward": 4.599987611174583,
-      "reward_std": 1.3417337444698205,
-      "rewards/concensus_correctness_reward_func": 1.2383124865591526,
-      "rewards/consensus_reward_func": 1.4375,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.7561751045286655,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.203125,
-      "rewards/xmlcount_reward_func": 0.9648750014603138,
+      "loss": 0.0264,
+      "num_tokens": 68200.0,
+      "reward": 0.15735165495425463,
+      "reward_std": 0.20545156858861446,
+      "rewards/concensus_correctness_reward_func/mean": 0.015625,
+      "rewards/concensus_correctness_reward_func/std": 0.04419417306780815,
+      "rewards/consensus_reward_func/mean": 0.125,
+      "rewards/consensus_reward_func/std": 0.3535533845424652,
+      "rewards/cumulative_reward_2/mean": 0.0,
+      "rewards/cumulative_reward_2/std": 0.0,
+      "rewards/final_correctness_reward_func/mean": 0.0,
+      "rewards/final_correctness_reward_func/std": 0.0,
+      "rewards/question_recreation_reward_func/mean": 0.016726648900657892,
+      "rewards/question_recreation_reward_func/std": 0.010374929523095489,
+      "rewards/soft_format_reward_func/mean": 0.0,
+      "rewards/soft_format_reward_func/std": 0.0,
+      "rewards/strict_format_reward_func/mean": 0.0,
+      "rewards/strict_format_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": 0.0,
+      "rewards/xmlcount_reward_func/std": 0.0,
       "step": 14
     },
     {
-      "completion_length": 165.45833333333334,
-      "epoch": 4.0,
-      "grad_norm": 884.29345703125,
-      "kl": 153.8786713940402,
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0625,
+      "completions/max_length": 749.5,
+      "completions/max_terminated_length": 605.5,
+      "completions/mean_length": 282.5,
+      "completions/mean_terminated_length": 231.04464721679688,
+      "completions/min_length": 12.5,
+      "completions/min_terminated_length": 12.5,
+      "epoch": 0.7710843373493976,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 6.5093994140625,
+      "kl": 0.002072617062367499,
       "learning_rate": 8.067960709356478e-08,
-      "loss": 0.1154,
-      "reward": 5.407021721204122,
-      "reward_std": 0.9267256280872971,
-      "rewards/concensus_correctness_reward_func": 1.5748333086570103,
-      "rewards/consensus_reward_func": 1.5,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.7176050413399935,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.1354166666666667,
+      "loss": 0.0046,
+      "num_tokens": 76816.0,
+      "reward": 0.017174751963466406,
+      "reward_std": 0.011310524307191372,
+      "rewards/concensus_correctness_reward_func/mean": 0.0,
+      "rewards/concensus_correctness_reward_func/std": 0.0,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
+      "rewards/cumulative_reward_2/mean": 0.0,
+      "rewards/cumulative_reward_2/std": 0.0,
+      "rewards/final_correctness_reward_func/mean": 0.0,
+      "rewards/final_correctness_reward_func/std": 0.0,
+      "rewards/question_recreation_reward_func/mean": 0.017174751963466406,
+      "rewards/question_recreation_reward_func/std": 0.011285829357802868,
+      "rewards/soft_format_reward_func/mean": 0.0,
+      "rewards/soft_format_reward_func/std": 0.0,
+      "rewards/strict_format_reward_func/mean": 0.0,
+      "rewards/strict_format_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": 0.0,
+      "rewards/xmlcount_reward_func/std": 0.0,
       "step": 16
     },
     {
-      "completion_length": 167.03125,
-      "epoch": 4.571428571428571,
-      "grad_norm": 433.138916015625,
-      "kl": 51.80729316617362,
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 941.5,
+      "completions/max_terminated_length": 941.5,
+      "completions/mean_length": 366.375,
+      "completions/mean_terminated_length": 366.375,
+      "completions/min_length": 4.5,
+      "completions/min_terminated_length": 4.5,
+      "epoch": 0.8674698795180723,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 4.666317939758301,
+      "kl": 0.0007759865002299193,
       "learning_rate": 3.013156219837776e-08,
-      "loss": 0.0518,
-      "reward": 4.95432373136282,
-      "reward_std": 0.6955515777153778,
-      "rewards/concensus_correctness_reward_func": 1.3982499837875366,
-      "rewards/consensus_reward_func": 1.5,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.7722300551831722,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.25,
-      "rewards/xmlcount_reward_func": 1.0338437519967556,
+      "loss": 0.1212,
+      "num_tokens": 86774.0,
+      "reward": -0.03930019214749336,
+      "reward_std": 0.15650326944887638,
+      "rewards/concensus_correctness_reward_func/mean": 0.0,
+      "rewards/concensus_correctness_reward_func/std": 0.0,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
+      "rewards/cumulative_reward_2/mean": 0.0,
+      "rewards/cumulative_reward_2/std": 0.0,
+      "rewards/final_correctness_reward_func/mean": 0.0,
+      "rewards/final_correctness_reward_func/std": 0.0,
+      "rewards/question_recreation_reward_func/mean": 0.05757479928433895,
+      "rewards/question_recreation_reward_func/std": 0.03582485252991319,
+      "rewards/soft_format_reward_func/mean": 0.0,
+      "rewards/soft_format_reward_func/std": 0.0,
+      "rewards/strict_format_reward_func/mean": 0.0,
+      "rewards/strict_format_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": -0.09687499701976776,
+      "rewards/xmlcount_reward_func/std": 0.27400386333465576,
       "step": 18
     },
     {
-      "completion_length": 213.45833333333334,
-      "epoch": 5.0,
-      "grad_norm": 30.303447723388672,
-      "kl": 1426.8845755159855,
+      "clip_ratio/high_max": 0.0,
+      "clip_ratio/high_mean": 0.0,
+      "clip_ratio/low_mean": 0.0,
+      "clip_ratio/low_min": 0.0,
+      "clip_ratio/region_mean": 0.0,
+      "completions/clipped_ratio": 0.1875,
+      "completions/max_length": 928.0,
+      "completions/max_terminated_length": 814.0,
+      "completions/mean_length": 394.0625,
+      "completions/mean_terminated_length": 260.71250915527344,
+      "completions/min_length": 4.5,
+      "completions/min_terminated_length": 4.5,
+      "epoch": 0.963855421686747,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 11.774459838867188,
+      "kl": 0.001760888408171013,
       "learning_rate": 3.4096741493194193e-09,
-      "loss": 1.0702,
-      "reward": 4.796473105748494,
-      "reward_std": 0.7849306451777617,
-      "rewards/concensus_correctness_reward_func": 1.431666652361552,
-      "rewards/consensus_reward_func": 1.4166666666666667,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.7149731454749902,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.25,
-      "rewards/xmlcount_reward_func": 0.9831666549046835,
+      "loss": -0.0662,
+      "num_tokens": 97175.0,
+      "reward": 0.017644216306507587,
+      "reward_std": 0.008533301530405879,
+      "rewards/concensus_correctness_reward_func/mean": 0.0,
+      "rewards/concensus_correctness_reward_func/std": 0.0,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
+      "rewards/cumulative_reward_2/mean": 0.0,
+      "rewards/cumulative_reward_2/std": 0.0,
+      "rewards/final_correctness_reward_func/mean": 0.0,
+      "rewards/final_correctness_reward_func/std": 0.0,
+      "rewards/question_recreation_reward_func/mean": 0.017644216306507587,
+      "rewards/question_recreation_reward_func/std": 0.014659160049632192,
+      "rewards/soft_format_reward_func/mean": 0.0,
+      "rewards/soft_format_reward_func/std": 0.0,
+      "rewards/strict_format_reward_func/mean": 0.0,
+      "rewards/strict_format_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": 0.0,
+      "rewards/xmlcount_reward_func/std": 0.0,
       "step": 20
     },
     {
-      "epoch": 5.0,
+      "epoch": 0.963855421686747,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 0.12956260753126117,
-      "train_runtime": 123.8534,
-      "train_samples_per_second": 2.584,
-      "train_steps_per_second": 0.161
+      "train_loss": 0.03613492660224438,
+      "train_runtime": 1374.3618,
+      "train_samples_per_second": 0.116,
+      "train_steps_per_second": 0.015
     }
   ],
   "logging_steps": 2,
   "max_steps": 20,
-  "num_input_tokens_seen": 0,
-  "num_train_epochs": 7,
+  "num_input_tokens_seen": 97175,
+  "num_train_epochs": 1,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:15a35ed0dc1b427b3d92ea468297276e7e5514301a2a4c403cab4aa16463d2e6
-size 5944
+oid sha256:6bc2740e757e02dffaef9c67456e365b5d93e3b713615b66406fede2cc76013c
+size 6929