Model save

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,58 @@
+---
+base_model: Qwen/Qwen2.5-1.5B-Instruct
+library_name: transformers
+model_name: Qwen2.5-1.5B-Open-R1-Distill
+tags:
+- generated_from_trainer
+- trl
+- sft
+licence: license
+---
+
+# Model Card for Qwen2.5-1.5B-Open-R1-Distill
+
+This model is a fine-tuned version of [Qwen/Qwen2.5-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="flyingbugs/Qwen2.5-1.5B-Open-R1-Distill", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/jjh233/huggingface/runs/mt4uvcrz) 
+
+
+This model was trained with SFT.
+
+### Framework versions
+
+- TRL: 0.16.0.dev0
+- Transformers: 4.49.0.dev0
+- Pytorch: 2.5.1
+- Datasets: 3.3.0
+- Tokenizers: 0.21.0
+
+## Citations
+
+
+
+Cite TRL as:
+    
+```bibtex
+@misc{vonwerra2022trl,
+	title        = {{TRL: Transformer Reinforcement Learning}},
+	author       = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallouédec},
+	year         = 2020,
+	journal      = {GitHub repository},
+	publisher    = {GitHub},
+	howpublished = {\url{https://github.com/huggingface/trl}}
+}
+```

added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

all_results.json

@@ -0,0 +1,8 @@
+{
+    "total_flos": 6.960985517882081e+17,
+    "train_loss": 0.8210960451450108,
+    "train_runtime": 16213.9925,
+    "train_samples": 16610,
+    "train_samples_per_second": 1.334,
+    "train_steps_per_second": 0.083
+}

config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "Qwen/Qwen2.5-1.5B-Instruct",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 1536,
+  "initializer_range": 0.02,
+  "intermediate_size": 8960,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 21,
+  "model_type": "qwen2",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": 32768,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.49.0.dev0",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.1,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "4.49.0.dev0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b947d0033b51db576da05474b5ab165f05167b2332af19abb0f0ac5c7a178874
+size 3087467144

special_tokens_map.json

@@ -0,0 +1,25 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|im_end|>"
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
+size 11421896

tokenizer_config.json

@@ -0,0 +1,208 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|im_end|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "total_flos": 6.960985517882081e+17,
+    "train_loss": 0.8210960451450108,
+    "train_runtime": 16213.9925,
+    "train_samples": 16610,
+    "train_samples_per_second": 1.334,
+    "train_steps_per_second": 0.083
+}

trainer_state.json

@@ -0,0 +1,2320 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9996300406955235,
+  "eval_steps": 100,
+  "global_step": 1351,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.003699593044765076,
+      "grad_norm": 2.75,
+      "learning_rate": 7.352941176470589e-07,
+      "loss": 1.1066,
+      "mean_token_accuracy": 0.7088243922791502,
+      "step": 5
+    },
+    {
+      "epoch": 0.007399186089530152,
+      "grad_norm": 2.828125,
+      "learning_rate": 1.4705882352941177e-06,
+      "loss": 1.0707,
+      "mean_token_accuracy": 0.7166337003356005,
+      "step": 10
+    },
+    {
+      "epoch": 0.011098779134295227,
+      "grad_norm": 2.671875,
+      "learning_rate": 2.2058823529411767e-06,
+      "loss": 1.1384,
+      "mean_token_accuracy": 0.6971814349668566,
+      "step": 15
+    },
+    {
+      "epoch": 0.014798372179060304,
+      "grad_norm": 2.53125,
+      "learning_rate": 2.9411764705882355e-06,
+      "loss": 1.0498,
+      "mean_token_accuracy": 0.7220594752142167,
+      "step": 20
+    },
+    {
+      "epoch": 0.01849796522382538,
+      "grad_norm": 2.265625,
+      "learning_rate": 3.6764705882352946e-06,
+      "loss": 1.1215,
+      "mean_token_accuracy": 0.7044319839717564,
+      "step": 25
+    },
+    {
+      "epoch": 0.022197558268590455,
+      "grad_norm": 1.953125,
+      "learning_rate": 4.411764705882353e-06,
+      "loss": 1.0543,
+      "mean_token_accuracy": 0.7190543555058295,
+      "step": 30
+    },
+    {
+      "epoch": 0.02589715131335553,
+      "grad_norm": 2.0,
+      "learning_rate": 5.147058823529411e-06,
+      "loss": 1.0971,
+      "mean_token_accuracy": 0.7085486170091443,
+      "step": 35
+    },
+    {
+      "epoch": 0.029596744358120607,
+      "grad_norm": 1.5546875,
+      "learning_rate": 5.882352941176471e-06,
+      "loss": 1.0428,
+      "mean_token_accuracy": 0.7209126843652592,
+      "step": 40
+    },
+    {
+      "epoch": 0.033296337402885685,
+      "grad_norm": 1.4140625,
+      "learning_rate": 6.61764705882353e-06,
+      "loss": 1.0191,
+      "mean_token_accuracy": 0.7255064573683462,
+      "step": 45
+    },
+    {
+      "epoch": 0.03699593044765076,
+      "grad_norm": 1.4140625,
+      "learning_rate": 7.352941176470589e-06,
+      "loss": 1.0063,
+      "mean_token_accuracy": 0.7245021778659816,
+      "step": 50
+    },
+    {
+      "epoch": 0.040695523492415835,
+      "grad_norm": 1.25,
+      "learning_rate": 8.088235294117648e-06,
+      "loss": 0.9647,
+      "mean_token_accuracy": 0.7356470962748933,
+      "step": 55
+    },
+    {
+      "epoch": 0.04439511653718091,
+      "grad_norm": 1.1796875,
+      "learning_rate": 8.823529411764707e-06,
+      "loss": 0.9878,
+      "mean_token_accuracy": 0.7297646791302468,
+      "step": 60
+    },
+    {
+      "epoch": 0.048094709581945984,
+      "grad_norm": 1.0234375,
+      "learning_rate": 9.558823529411766e-06,
+      "loss": 0.9923,
+      "mean_token_accuracy": 0.7247109358252525,
+      "step": 65
+    },
+    {
+      "epoch": 0.05179430262671106,
+      "grad_norm": 1.15625,
+      "learning_rate": 1.0294117647058823e-05,
+      "loss": 0.991,
+      "mean_token_accuracy": 0.7247228263454575,
+      "step": 70
+    },
+    {
+      "epoch": 0.05549389567147614,
+      "grad_norm": 1.2109375,
+      "learning_rate": 1.1029411764705885e-05,
+      "loss": 0.981,
+      "mean_token_accuracy": 0.7259009298847336,
+      "step": 75
+    },
+    {
+      "epoch": 0.059193488716241215,
+      "grad_norm": 0.96875,
+      "learning_rate": 1.1764705882352942e-05,
+      "loss": 0.9264,
+      "mean_token_accuracy": 0.7393706865844448,
+      "step": 80
+    },
+    {
+      "epoch": 0.06289308176100629,
+      "grad_norm": 0.91796875,
+      "learning_rate": 1.25e-05,
+      "loss": 0.9213,
+      "mean_token_accuracy": 0.7397875887407016,
+      "step": 85
+    },
+    {
+      "epoch": 0.06659267480577137,
+      "grad_norm": 1.046875,
+      "learning_rate": 1.323529411764706e-05,
+      "loss": 0.935,
+      "mean_token_accuracy": 0.7356831547759819,
+      "step": 90
+    },
+    {
+      "epoch": 0.07029226785053644,
+      "grad_norm": 1.0234375,
+      "learning_rate": 1.3970588235294118e-05,
+      "loss": 0.8699,
+      "mean_token_accuracy": 0.7526627418636163,
+      "step": 95
+    },
+    {
+      "epoch": 0.07399186089530152,
+      "grad_norm": 1.0625,
+      "learning_rate": 1.4705882352941179e-05,
+      "loss": 0.9213,
+      "mean_token_accuracy": 0.737186059951846,
+      "step": 100
+    },
+    {
+      "epoch": 0.07399186089530152,
+      "eval_loss": 0.9279058575630188,
+      "eval_mean_token_accuracy": 0.7344749832147877,
+      "eval_runtime": 27.0967,
+      "eval_samples_per_second": 4.761,
+      "eval_steps_per_second": 1.218,
+      "step": 100
+    },
+    {
+      "epoch": 0.07769145394006659,
+      "grad_norm": 0.96484375,
+      "learning_rate": 1.5441176470588237e-05,
+      "loss": 0.9404,
+      "mean_token_accuracy": 0.7320285350313906,
+      "step": 105
+    },
+    {
+      "epoch": 0.08139104698483167,
+      "grad_norm": 0.92578125,
+      "learning_rate": 1.6176470588235296e-05,
+      "loss": 0.8782,
+      "mean_token_accuracy": 0.7472598345781812,
+      "step": 110
+    },
+    {
+      "epoch": 0.08509064002959675,
+      "grad_norm": 0.8984375,
+      "learning_rate": 1.6911764705882355e-05,
+      "loss": 0.9146,
+      "mean_token_accuracy": 0.7384105813781348,
+      "step": 115
+    },
+    {
+      "epoch": 0.08879023307436182,
+      "grad_norm": 0.9609375,
+      "learning_rate": 1.7647058823529414e-05,
+      "loss": 0.8929,
+      "mean_token_accuracy": 0.746422576625229,
+      "step": 120
+    },
+    {
+      "epoch": 0.0924898261191269,
+      "grad_norm": 1.0234375,
+      "learning_rate": 1.8382352941176472e-05,
+      "loss": 0.91,
+      "mean_token_accuracy": 0.7382566905333322,
+      "step": 125
+    },
+    {
+      "epoch": 0.09618941916389197,
+      "grad_norm": 0.94140625,
+      "learning_rate": 1.911764705882353e-05,
+      "loss": 0.864,
+      "mean_token_accuracy": 0.75168947141329,
+      "step": 130
+    },
+    {
+      "epoch": 0.09988901220865705,
+      "grad_norm": 0.875,
+      "learning_rate": 1.985294117647059e-05,
+      "loss": 0.8434,
+      "mean_token_accuracy": 0.7577931647267211,
+      "step": 135
+    },
+    {
+      "epoch": 0.10358860525342212,
+      "grad_norm": 0.91015625,
+      "learning_rate": 1.9999465148392906e-05,
+      "loss": 0.8517,
+      "mean_token_accuracy": 0.753664157035871,
+      "step": 140
+    },
+    {
+      "epoch": 0.1072881982981872,
+      "grad_norm": 1.0078125,
+      "learning_rate": 1.999729241179462e-05,
+      "loss": 0.8444,
+      "mean_token_accuracy": 0.7564269476656776,
+      "step": 145
+    },
+    {
+      "epoch": 0.11098779134295228,
+      "grad_norm": 0.8515625,
+      "learning_rate": 1.999344872485215e-05,
+      "loss": 0.846,
+      "mean_token_accuracy": 0.7548579441013794,
+      "step": 150
+    },
+    {
+      "epoch": 0.11468738438771735,
+      "grad_norm": 1.03125,
+      "learning_rate": 1.9987934730000457e-05,
+      "loss": 0.8415,
+      "mean_token_accuracy": 0.7574775187410825,
+      "step": 155
+    },
+    {
+      "epoch": 0.11838697743248243,
+      "grad_norm": 1.03125,
+      "learning_rate": 1.998075134885022e-05,
+      "loss": 0.8541,
+      "mean_token_accuracy": 0.752040333685926,
+      "step": 160
+    },
+    {
+      "epoch": 0.1220865704772475,
+      "grad_norm": 1.140625,
+      "learning_rate": 1.9971899782033853e-05,
+      "loss": 0.849,
+      "mean_token_accuracy": 0.7540821334441492,
+      "step": 165
+    },
+    {
+      "epoch": 0.12578616352201258,
+      "grad_norm": 0.9375,
+      "learning_rate": 1.9961381509004785e-05,
+      "loss": 0.8585,
+      "mean_token_accuracy": 0.750343948398876,
+      "step": 170
+    },
+    {
+      "epoch": 0.12948575656677766,
+      "grad_norm": 0.92578125,
+      "learning_rate": 1.9949198287790215e-05,
+      "loss": 0.8344,
+      "mean_token_accuracy": 0.7581955679383152,
+      "step": 175
+    },
+    {
+      "epoch": 0.13318534961154274,
+      "grad_norm": 0.9140625,
+      "learning_rate": 1.9935352154697257e-05,
+      "loss": 0.855,
+      "mean_token_accuracy": 0.75095165766086,
+      "step": 180
+    },
+    {
+      "epoch": 0.1368849426563078,
+      "grad_norm": 0.875,
+      "learning_rate": 1.9919845423972603e-05,
+      "loss": 0.806,
+      "mean_token_accuracy": 0.7657711359955364,
+      "step": 185
+    },
+    {
+      "epoch": 0.14058453570107288,
+      "grad_norm": 1.0625,
+      "learning_rate": 1.9902680687415704e-05,
+      "loss": 0.8465,
+      "mean_token_accuracy": 0.7529327383820095,
+      "step": 190
+    },
+    {
+      "epoch": 0.14428412874583796,
+      "grad_norm": 1.0703125,
+      "learning_rate": 1.9883860813945596e-05,
+      "loss": 0.847,
+      "mean_token_accuracy": 0.7524867088122604,
+      "step": 195
+    },
+    {
+      "epoch": 0.14798372179060304,
+      "grad_norm": 0.96484375,
+      "learning_rate": 1.986338894912137e-05,
+      "loss": 0.8333,
+      "mean_token_accuracy": 0.7572265455201546,
+      "step": 200
+    },
+    {
+      "epoch": 0.14798372179060304,
+      "eval_loss": 0.8691926598548889,
+      "eval_mean_token_accuracy": 0.7449847329506468,
+      "eval_runtime": 26.8448,
+      "eval_samples_per_second": 4.805,
+      "eval_steps_per_second": 1.229,
+      "step": 200
+    },
+    {
+      "epoch": 0.15168331483536812,
+      "grad_norm": 0.859375,
+      "learning_rate": 1.9841268514616434e-05,
+      "loss": 0.8675,
+      "mean_token_accuracy": 0.7473276412330847,
+      "step": 205
+    },
+    {
+      "epoch": 0.15538290788013318,
+      "grad_norm": 0.8359375,
+      "learning_rate": 1.9817503207646606e-05,
+      "loss": 0.8303,
+      "mean_token_accuracy": 0.7578632356560819,
+      "step": 210
+    },
+    {
+      "epoch": 0.15908250092489826,
+      "grad_norm": 0.953125,
+      "learning_rate": 1.979209700035216e-05,
+      "loss": 0.8566,
+      "mean_token_accuracy": 0.7511934513470362,
+      "step": 215
+    },
+    {
+      "epoch": 0.16278209396966334,
+      "grad_norm": 0.94140625,
+      "learning_rate": 1.976505413913393e-05,
+      "loss": 0.8191,
+      "mean_token_accuracy": 0.7605520317739141,
+      "step": 220
+    },
+    {
+      "epoch": 0.16648168701442842,
+      "grad_norm": 0.98828125,
+      "learning_rate": 1.9736379143943565e-05,
+      "loss": 0.8263,
+      "mean_token_accuracy": 0.7587204409982755,
+      "step": 225
+    },
+    {
+      "epoch": 0.1701812800591935,
+      "grad_norm": 0.87109375,
+      "learning_rate": 1.9706076807528044e-05,
+      "loss": 0.8249,
+      "mean_token_accuracy": 0.7590353833631612,
+      "step": 230
+    },
+    {
+      "epoch": 0.17388087310395856,
+      "grad_norm": 1.0546875,
+      "learning_rate": 1.967415219462864e-05,
+      "loss": 0.8215,
+      "mean_token_accuracy": 0.7610956831994506,
+      "step": 235
+    },
+    {
+      "epoch": 0.17758046614872364,
+      "grad_norm": 0.921875,
+      "learning_rate": 1.9640610641134383e-05,
+      "loss": 0.8574,
+      "mean_token_accuracy": 0.7499589429813166,
+      "step": 240
+    },
+    {
+      "epoch": 0.18128005919348872,
+      "grad_norm": 0.8984375,
+      "learning_rate": 1.9605457753190224e-05,
+      "loss": 0.8206,
+      "mean_token_accuracy": 0.7588238725869608,
+      "step": 245
+    },
+    {
+      "epoch": 0.1849796522382538,
+      "grad_norm": 0.8515625,
+      "learning_rate": 1.9568699406260016e-05,
+      "loss": 0.8068,
+      "mean_token_accuracy": 0.762297568145403,
+      "step": 250
+    },
+    {
+      "epoch": 0.18867924528301888,
+      "grad_norm": 0.83203125,
+      "learning_rate": 1.953034174414449e-05,
+      "loss": 0.8514,
+      "mean_token_accuracy": 0.7505548580731571,
+      "step": 255
+    },
+    {
+      "epoch": 0.19237883832778394,
+      "grad_norm": 0.953125,
+      "learning_rate": 1.9490391177954383e-05,
+      "loss": 0.8145,
+      "mean_token_accuracy": 0.7608352213106663,
+      "step": 260
+    },
+    {
+      "epoch": 0.19607843137254902,
+      "grad_norm": 0.8515625,
+      "learning_rate": 1.944885438503888e-05,
+      "loss": 0.831,
+      "mean_token_accuracy": 0.7539169665265308,
+      "step": 265
+    },
+    {
+      "epoch": 0.1997780244173141,
+      "grad_norm": 0.86328125,
+      "learning_rate": 1.9405738307869565e-05,
+      "loss": 0.8031,
+      "mean_token_accuracy": 0.7646443725758238,
+      "step": 270
+    },
+    {
+      "epoch": 0.20347761746207918,
+      "grad_norm": 0.91796875,
+      "learning_rate": 1.936105015288003e-05,
+      "loss": 0.833,
+      "mean_token_accuracy": 0.7572038567258342,
+      "step": 275
+    },
+    {
+      "epoch": 0.20717721050684423,
+      "grad_norm": 0.9453125,
+      "learning_rate": 1.9314797389261426e-05,
+      "loss": 0.8418,
+      "mean_token_accuracy": 0.7531908117025352,
+      "step": 280
+    },
+    {
+      "epoch": 0.21087680355160932,
+      "grad_norm": 1.0078125,
+      "learning_rate": 1.9266987747714036e-05,
+      "loss": 0.793,
+      "mean_token_accuracy": 0.7670622612808323,
+      "step": 285
+    },
+    {
+      "epoch": 0.2145763965963744,
+      "grad_norm": 0.89453125,
+      "learning_rate": 1.9217629219155172e-05,
+      "loss": 0.8032,
+      "mean_token_accuracy": 0.763842202116048,
+      "step": 290
+    },
+    {
+      "epoch": 0.21827598964113948,
+      "grad_norm": 0.875,
+      "learning_rate": 1.916673005338357e-05,
+      "loss": 0.8097,
+      "mean_token_accuracy": 0.7612816649178271,
+      "step": 295
+    },
+    {
+      "epoch": 0.22197558268590456,
+      "grad_norm": 0.91796875,
+      "learning_rate": 1.9114298757700508e-05,
+      "loss": 0.8289,
+      "mean_token_accuracy": 0.7554124302032139,
+      "step": 300
+    },
+    {
+      "epoch": 0.22197558268590456,
+      "eval_loss": 0.8505480885505676,
+      "eval_mean_token_accuracy": 0.7487433402846764,
+      "eval_runtime": 26.8365,
+      "eval_samples_per_second": 4.807,
+      "eval_steps_per_second": 1.23,
+      "step": 300
+    },
+    {
+      "epoch": 0.2256751757306696,
+      "grad_norm": 0.99609375,
+      "learning_rate": 1.9060344095487916e-05,
+      "loss": 0.7957,
+      "mean_token_accuracy": 0.7660681762120751,
+      "step": 305
+    },
+    {
+      "epoch": 0.2293747687754347,
+      "grad_norm": 0.96875,
+      "learning_rate": 1.9004875084743624e-05,
+      "loss": 0.8214,
+      "mean_token_accuracy": 0.7592361950375772,
+      "step": 310
+    },
+    {
+      "epoch": 0.23307436182019978,
+      "grad_norm": 0.953125,
+      "learning_rate": 1.8947900996574133e-05,
+      "loss": 0.8306,
+      "mean_token_accuracy": 0.756349200401793,
+      "step": 315
+    },
+    {
+      "epoch": 0.23677395486496486,
+      "grad_norm": 0.86328125,
+      "learning_rate": 1.8889431353645004e-05,
+      "loss": 0.8289,
+      "mean_token_accuracy": 0.7558863317625434,
+      "step": 320
+    },
+    {
+      "epoch": 0.24047354790972994,
+      "grad_norm": 0.83984375,
+      "learning_rate": 1.8829475928589272e-05,
+      "loss": 0.7879,
+      "mean_token_accuracy": 0.7676201433765534,
+      "step": 325
+    },
+    {
+      "epoch": 0.244173140954495,
+      "grad_norm": 0.91015625,
+      "learning_rate": 1.8768044742374008e-05,
+      "loss": 0.828,
+      "mean_token_accuracy": 0.7566986375040194,
+      "step": 330
+    },
+    {
+      "epoch": 0.24787273399926008,
+      "grad_norm": 1.0390625,
+      "learning_rate": 1.870514806262544e-05,
+      "loss": 0.8521,
+      "mean_token_accuracy": 0.7493918412218303,
+      "step": 335
+    },
+    {
+      "epoch": 0.25157232704402516,
+      "grad_norm": 0.859375,
+      "learning_rate": 1.8640796401912805e-05,
+      "loss": 0.8238,
+      "mean_token_accuracy": 0.7569454376269539,
+      "step": 340
+    },
+    {
+      "epoch": 0.25527192008879024,
+      "grad_norm": 0.91015625,
+      "learning_rate": 1.8575000515991283e-05,
+      "loss": 0.7741,
+      "mean_token_accuracy": 0.7709062370742213,
+      "step": 345
+    },
+    {
+      "epoch": 0.2589715131335553,
+      "grad_norm": 0.95703125,
+      "learning_rate": 1.850777140200427e-05,
+      "loss": 0.7908,
+      "mean_token_accuracy": 0.7679519537903451,
+      "step": 350
+    },
+    {
+      "epoch": 0.2626711061783204,
+      "grad_norm": 0.9140625,
+      "learning_rate": 1.843912029664531e-05,
+      "loss": 0.8102,
+      "mean_token_accuracy": 0.7614762706916716,
+      "step": 355
+    },
+    {
+      "epoch": 0.2663706992230855,
+      "grad_norm": 0.984375,
+      "learning_rate": 1.8369058674280004e-05,
+      "loss": 0.8188,
+      "mean_token_accuracy": 0.7584292346155446,
+      "step": 360
+    },
+    {
+      "epoch": 0.2700702922678505,
+      "grad_norm": 0.85546875,
+      "learning_rate": 1.8297598245028173e-05,
+      "loss": 0.822,
+      "mean_token_accuracy": 0.7582656406624906,
+      "step": 365
+    },
+    {
+      "epoch": 0.2737698853126156,
+      "grad_norm": 0.99609375,
+      "learning_rate": 1.8224750952806626e-05,
+      "loss": 0.8051,
+      "mean_token_accuracy": 0.7627783319750381,
+      "step": 370
+    },
+    {
+      "epoch": 0.27746947835738067,
+      "grad_norm": 0.875,
+      "learning_rate": 1.815052897333284e-05,
+      "loss": 0.7961,
+      "mean_token_accuracy": 0.7659223247619174,
+      "step": 375
+    },
+    {
+      "epoch": 0.28116907140214575,
+      "grad_norm": 0.90625,
+      "learning_rate": 1.8074944712089925e-05,
+      "loss": 0.8073,
+      "mean_token_accuracy": 0.761887104957649,
+      "step": 380
+    },
+    {
+      "epoch": 0.28486866444691084,
+      "grad_norm": 0.92578125,
+      "learning_rate": 1.799801080225316e-05,
+      "loss": 0.7964,
+      "mean_token_accuracy": 0.7628394476937959,
+      "step": 385
+    },
+    {
+      "epoch": 0.2885682574916759,
+      "grad_norm": 0.87890625,
+      "learning_rate": 1.7919740102578482e-05,
+      "loss": 0.8217,
+      "mean_token_accuracy": 0.757059202645602,
+      "step": 390
+    },
+    {
+      "epoch": 0.292267850536441,
+      "grad_norm": 0.94921875,
+      "learning_rate": 1.7840145695253258e-05,
+      "loss": 0.8178,
+      "mean_token_accuracy": 0.7587981397498871,
+      "step": 395
+    },
+    {
+      "epoch": 0.2959674435812061,
+      "grad_norm": 0.8984375,
+      "learning_rate": 1.7759240883709745e-05,
+      "loss": 0.8074,
+      "mean_token_accuracy": 0.762638185984242,
+      "step": 400
+    },
+    {
+      "epoch": 0.2959674435812061,
+      "eval_loss": 0.8404825925827026,
+      "eval_mean_token_accuracy": 0.7507319702479877,
+      "eval_runtime": 26.8462,
+      "eval_samples_per_second": 4.805,
+      "eval_steps_per_second": 1.229,
+      "step": 400
+    },
+    {
+      "epoch": 0.29966703662597116,
+      "grad_norm": 0.84765625,
+      "learning_rate": 1.7677039190401538e-05,
+      "loss": 0.8429,
+      "mean_token_accuracy": 0.7518463419239729,
+      "step": 405
+    },
+    {
+      "epoch": 0.30336662967073624,
+      "grad_norm": 0.85546875,
+      "learning_rate": 1.759355435454342e-05,
+      "loss": 0.8018,
+      "mean_token_accuracy": 0.7638060805093954,
+      "step": 410
+    },
+    {
+      "epoch": 0.30706622271550127,
+      "grad_norm": 0.859375,
+      "learning_rate": 1.7508800329814993e-05,
+      "loss": 0.8031,
+      "mean_token_accuracy": 0.7625890814886728,
+      "step": 415
+    },
+    {
+      "epoch": 0.31076581576026635,
+      "grad_norm": 0.859375,
+      "learning_rate": 1.7422791282028457e-05,
+      "loss": 0.7891,
+      "mean_token_accuracy": 0.7665874312175128,
+      "step": 420
+    },
+    {
+      "epoch": 0.31446540880503143,
+      "grad_norm": 0.8984375,
+      "learning_rate": 1.7335541586760928e-05,
+      "loss": 0.8148,
+      "mean_token_accuracy": 0.7591026778855703,
+      "step": 425
+    },
+    {
+      "epoch": 0.3181650018497965,
+      "grad_norm": 1.03125,
+      "learning_rate": 1.7247065826951694e-05,
+      "loss": 0.8143,
+      "mean_token_accuracy": 0.757982232348984,
+      "step": 430
+    },
+    {
+      "epoch": 0.3218645948945616,
+      "grad_norm": 0.8984375,
+      "learning_rate": 1.715737879046483e-05,
+      "loss": 0.8276,
+      "mean_token_accuracy": 0.7568147231068972,
+      "step": 435
+    },
+    {
+      "epoch": 0.3255641879393267,
+      "grad_norm": 0.91015625,
+      "learning_rate": 1.7066495467617552e-05,
+      "loss": 0.7656,
+      "mean_token_accuracy": 0.7725881889254189,
+      "step": 440
+    },
+    {
+      "epoch": 0.32926378098409176,
+      "grad_norm": 1.125,
+      "learning_rate": 1.6974431048674714e-05,
+      "loss": 0.8267,
+      "mean_token_accuracy": 0.7573400297502496,
+      "step": 445
+    },
+    {
+      "epoch": 0.33296337402885684,
+      "grad_norm": 0.78515625,
+      "learning_rate": 1.6881200921309914e-05,
+      "loss": 0.8266,
+      "mean_token_accuracy": 0.7558354919088875,
+      "step": 450
+    },
+    {
+      "epoch": 0.3366629670736219,
+      "grad_norm": 1.1171875,
+      "learning_rate": 1.6786820668033596e-05,
+      "loss": 0.8046,
+      "mean_token_accuracy": 0.7623769541100633,
+      "step": 455
+    },
+    {
+      "epoch": 0.340362560118387,
+      "grad_norm": 0.93359375,
+      "learning_rate": 1.6691306063588583e-05,
+      "loss": 0.8114,
+      "mean_token_accuracy": 0.7599912301268692,
+      "step": 460
+    },
+    {
+      "epoch": 0.34406215316315203,
+      "grad_norm": 0.95703125,
+      "learning_rate": 1.6594673072313478e-05,
+      "loss": 0.8031,
+      "mean_token_accuracy": 0.7619591903436009,
+      "step": 465
+    },
+    {
+      "epoch": 0.3477617462079171,
+      "grad_norm": 0.8515625,
+      "learning_rate": 1.6496937845474375e-05,
+      "loss": 0.8393,
+      "mean_token_accuracy": 0.7528874004871576,
+      "step": 470
+    },
+    {
+      "epoch": 0.3514613392526822,
+      "grad_norm": 0.8828125,
+      "learning_rate": 1.639811671856535e-05,
+      "loss": 0.8219,
+      "mean_token_accuracy": 0.7564342769297561,
+      "step": 475
+    },
+    {
+      "epoch": 0.3551609322974473,
+      "grad_norm": 0.859375,
+      "learning_rate": 1.6298226208578127e-05,
+      "loss": 0.8309,
+      "mean_token_accuracy": 0.7560339437248955,
+      "step": 480
+    },
+    {
+      "epoch": 0.35886052534221236,
+      "grad_norm": 0.8984375,
+      "learning_rate": 1.6197283011241423e-05,
+      "loss": 0.832,
+      "mean_token_accuracy": 0.7547866526401349,
+      "step": 485
+    },
+    {
+      "epoch": 0.36256011838697744,
+      "grad_norm": 0.87890625,
+      "learning_rate": 1.6095303998230432e-05,
+      "loss": 0.8147,
+      "mean_token_accuracy": 0.7593188336591906,
+      "step": 490
+    },
+    {
+      "epoch": 0.3662597114317425,
+      "grad_norm": 0.9375,
+      "learning_rate": 1.599230621434687e-05,
+      "loss": 0.8053,
+      "mean_token_accuracy": 0.7600767464825575,
+      "step": 495
+    },
+    {
+      "epoch": 0.3699593044765076,
+      "grad_norm": 0.9296875,
+      "learning_rate": 1.5888306874670112e-05,
+      "loss": 0.7818,
+      "mean_token_accuracy": 0.7679373163972162,
+      "step": 500
+    },
+    {
+      "epoch": 0.3699593044765076,
+      "eval_loss": 0.8337106704711914,
+      "eval_mean_token_accuracy": 0.7523667314963431,
+      "eval_runtime": 26.8438,
+      "eval_samples_per_second": 4.806,
+      "eval_steps_per_second": 1.229,
+      "step": 500
+    },
+    {
+      "epoch": 0.3736588975212727,
+      "grad_norm": 1.03125,
+      "learning_rate": 1.5783323361679865e-05,
+      "loss": 0.8291,
+      "mean_token_accuracy": 0.7555905929668592,
+      "step": 505
+    },
+    {
+      "epoch": 0.37735849056603776,
+      "grad_norm": 0.96484375,
+      "learning_rate": 1.567737322235084e-05,
+      "loss": 0.805,
+      "mean_token_accuracy": 0.7634317412279235,
+      "step": 510
+    },
+    {
+      "epoch": 0.3810580836108028,
+      "grad_norm": 0.875,
+      "learning_rate": 1.557047416521996e-05,
+      "loss": 0.7702,
+      "mean_token_accuracy": 0.7698003607653104,
+      "step": 515
+    },
+    {
+      "epoch": 0.38475767665556787,
+      "grad_norm": 0.90625,
+      "learning_rate": 1.546264405742654e-05,
+      "loss": 0.8282,
+      "mean_token_accuracy": 0.754502267738367,
+      "step": 520
+    },
+    {
+      "epoch": 0.38845726970033295,
+      "grad_norm": 0.828125,
+      "learning_rate": 1.535390092172597e-05,
+      "loss": 0.8172,
+      "mean_token_accuracy": 0.7586522103385217,
+      "step": 525
+    },
+    {
+      "epoch": 0.39215686274509803,
+      "grad_norm": 0.88671875,
+      "learning_rate": 1.5244262933477401e-05,
+      "loss": 0.8314,
+      "mean_token_accuracy": 0.7544460785619194,
+      "step": 530
+    },
+    {
+      "epoch": 0.3958564557898631,
+      "grad_norm": 0.95703125,
+      "learning_rate": 1.5133748417605878e-05,
+      "loss": 0.8238,
+      "mean_token_accuracy": 0.7560023308095679,
+      "step": 535
+    },
+    {
+      "epoch": 0.3995560488346282,
+      "grad_norm": 0.81640625,
+      "learning_rate": 1.5022375845539537e-05,
+      "loss": 0.7935,
+      "mean_token_accuracy": 0.7652901556383538,
+      "step": 540
+    },
+    {
+      "epoch": 0.4032556418793933,
+      "grad_norm": 0.90234375,
+      "learning_rate": 1.4910163832122278e-05,
+      "loss": 0.7745,
+      "mean_token_accuracy": 0.7703718223513919,
+      "step": 545
+    },
+    {
+      "epoch": 0.40695523492415836,
+      "grad_norm": 0.9921875,
+      "learning_rate": 1.4797131132502464e-05,
+      "loss": 0.7857,
+      "mean_token_accuracy": 0.7683202839029555,
+      "step": 550
+    },
+    {
+      "epoch": 0.41065482796892344,
+      "grad_norm": 0.94921875,
+      "learning_rate": 1.4683296638998192e-05,
+      "loss": 0.7983,
+      "mean_token_accuracy": 0.7629447694188413,
+      "step": 555
+    },
+    {
+      "epoch": 0.41435442101368847,
+      "grad_norm": 1.03125,
+      "learning_rate": 1.4568679377939619e-05,
+      "loss": 0.7849,
+      "mean_token_accuracy": 0.7667453318634945,
+      "step": 560
+    },
+    {
+      "epoch": 0.41805401405845355,
+      "grad_norm": 0.8828125,
+      "learning_rate": 1.4453298506488896e-05,
+      "loss": 0.7993,
+      "mean_token_accuracy": 0.7621667413651658,
+      "step": 565
+    },
+    {
+      "epoch": 0.42175360710321863,
+      "grad_norm": 0.85546875,
+      "learning_rate": 1.4337173309438236e-05,
+      "loss": 0.7937,
+      "mean_token_accuracy": 0.7640237723323762,
+      "step": 570
+    },
+    {
+      "epoch": 0.4254532001479837,
+      "grad_norm": 0.86328125,
+      "learning_rate": 1.4220323195986649e-05,
+      "loss": 0.7987,
+      "mean_token_accuracy": 0.7635135975883155,
+      "step": 575
+    },
+    {
+      "epoch": 0.4291527931927488,
+      "grad_norm": 0.92578125,
+      "learning_rate": 1.4102767696495885e-05,
+      "loss": 0.8174,
+      "mean_token_accuracy": 0.7587108280666874,
+      "step": 580
+    },
+    {
+      "epoch": 0.4328523862375139,
+      "grad_norm": 0.78125,
+      "learning_rate": 1.398452645922611e-05,
+      "loss": 0.818,
+      "mean_token_accuracy": 0.7588692861378166,
+      "step": 585
+    },
+    {
+      "epoch": 0.43655197928227896,
+      "grad_norm": 0.93359375,
+      "learning_rate": 1.3865619247051916e-05,
+      "loss": 0.7987,
+      "mean_token_accuracy": 0.7638109819722817,
+      "step": 590
+    },
+    {
+      "epoch": 0.44025157232704404,
+      "grad_norm": 0.8203125,
+      "learning_rate": 1.3746065934159123e-05,
+      "loss": 0.7734,
+      "mean_token_accuracy": 0.7709616798042245,
+      "step": 595
+    },
+    {
+      "epoch": 0.4439511653718091,
+      "grad_norm": 0.85546875,
+      "learning_rate": 1.3625886502723008e-05,
+      "loss": 0.8139,
+      "mean_token_accuracy": 0.7591778298409262,
+      "step": 600
+    },
+    {
+      "epoch": 0.4439511653718091,
+      "eval_loss": 0.829662561416626,
+      "eval_mean_token_accuracy": 0.7531342866047499,
+      "eval_runtime": 26.8378,
+      "eval_samples_per_second": 4.807,
+      "eval_steps_per_second": 1.23,
+      "step": 600
+    },
+    {
+      "epoch": 0.4476507584165742,
+      "grad_norm": 0.85546875,
+      "learning_rate": 1.3505101039568494e-05,
+      "loss": 0.7589,
+      "mean_token_accuracy": 0.7728507834722843,
+      "step": 605
+    },
+    {
+      "epoch": 0.4513503514613392,
+      "grad_norm": 0.83984375,
+      "learning_rate": 1.3383729732812814e-05,
+      "loss": 0.7831,
+      "mean_token_accuracy": 0.7667316695208302,
+      "step": 610
+    },
+    {
+      "epoch": 0.4550499445061043,
+      "grad_norm": 0.90234375,
+      "learning_rate": 1.3261792868491267e-05,
+      "loss": 0.8041,
+      "mean_token_accuracy": 0.7626523405292972,
+      "step": 615
+    },
+    {
+      "epoch": 0.4587495375508694,
+      "grad_norm": 0.92578125,
+      "learning_rate": 1.3139310827166613e-05,
+      "loss": 0.805,
+      "mean_token_accuracy": 0.7621301999883288,
+      "step": 620
+    },
+    {
+      "epoch": 0.46244913059563447,
+      "grad_norm": 0.875,
+      "learning_rate": 1.3016304080522657e-05,
+      "loss": 0.8344,
+      "mean_token_accuracy": 0.7525665110992816,
+      "step": 625
+    },
+    {
+      "epoch": 0.46614872364039955,
+      "grad_norm": 0.8359375,
+      "learning_rate": 1.2892793187942588e-05,
+      "loss": 0.8425,
+      "mean_token_accuracy": 0.7507631619825623,
+      "step": 630
+    },
+    {
+      "epoch": 0.46984831668516464,
+      "grad_norm": 0.86328125,
+      "learning_rate": 1.2768798793072708e-05,
+      "loss": 0.8147,
+      "mean_token_accuracy": 0.7586866730130766,
+      "step": 635
+    },
+    {
+      "epoch": 0.4735479097299297,
+      "grad_norm": 0.85546875,
+      "learning_rate": 1.2644341620372025e-05,
+      "loss": 0.7939,
+      "mean_token_accuracy": 0.7645759564508473,
+      "step": 640
+    },
+    {
+      "epoch": 0.4772475027746948,
+      "grad_norm": 0.8828125,
+      "learning_rate": 1.2519442471648364e-05,
+      "loss": 0.8072,
+      "mean_token_accuracy": 0.7615042486251242,
+      "step": 645
+    },
+    {
+      "epoch": 0.4809470958194599,
+      "grad_norm": 0.8359375,
+      "learning_rate": 1.2394122222581557e-05,
+      "loss": 0.7994,
+      "mean_token_accuracy": 0.7636894652301632,
+      "step": 650
+    },
+    {
+      "epoch": 0.48464668886422496,
+      "grad_norm": 0.89453125,
+      "learning_rate": 1.226840181923427e-05,
+      "loss": 0.7754,
+      "mean_token_accuracy": 0.7699820045986223,
+      "step": 655
+    },
+    {
+      "epoch": 0.48834628190899,
+      "grad_norm": 0.9609375,
+      "learning_rate": 1.214230227455106e-05,
+      "loss": 0.8151,
+      "mean_token_accuracy": 0.7586843050311536,
+      "step": 660
+    },
+    {
+      "epoch": 0.49204587495375507,
+      "grad_norm": 0.87109375,
+      "learning_rate": 1.201584466484629e-05,
+      "loss": 0.7948,
+      "mean_token_accuracy": 0.7642140264864977,
+      "step": 665
+    },
+    {
+      "epoch": 0.49574546799852015,
+      "grad_norm": 0.85546875,
+      "learning_rate": 1.1889050126281405e-05,
+      "loss": 0.8165,
+      "mean_token_accuracy": 0.7565695063597528,
+      "step": 670
+    },
+    {
+      "epoch": 0.49944506104328523,
+      "grad_norm": 0.88671875,
+      "learning_rate": 1.1761939851332241e-05,
+      "loss": 0.8273,
+      "mean_token_accuracy": 0.7559170471677257,
+      "step": 675
+    },
+    {
+      "epoch": 0.5031446540880503,
+      "grad_norm": 0.8671875,
+      "learning_rate": 1.1634535085246903e-05,
+      "loss": 0.809,
+      "mean_token_accuracy": 0.7606912141322844,
+      "step": 680
+    },
+    {
+      "epoch": 0.5068442471328154,
+      "grad_norm": 0.83984375,
+      "learning_rate": 1.1506857122494832e-05,
+      "loss": 0.8125,
+      "mean_token_accuracy": 0.7587615775769565,
+      "step": 685
+    },
+    {
+      "epoch": 0.5105438401775805,
+      "grad_norm": 0.828125,
+      "learning_rate": 1.1378927303207637e-05,
+      "loss": 0.8126,
+      "mean_token_accuracy": 0.7587384147705152,
+      "step": 690
+    },
+    {
+      "epoch": 0.5142434332223456,
+      "grad_norm": 0.88671875,
+      "learning_rate": 1.12507670096123e-05,
+      "loss": 0.7991,
+      "mean_token_accuracy": 0.7612190529857823,
+      "step": 695
+    },
+    {
+      "epoch": 0.5179430262671106,
+      "grad_norm": 0.98046875,
+      "learning_rate": 1.1122397662457352e-05,
+      "loss": 0.8033,
+      "mean_token_accuracy": 0.762910620256459,
+      "step": 700
+    },
+    {
+      "epoch": 0.5179430262671106,
+      "eval_loss": 0.8269818425178528,
+      "eval_mean_token_accuracy": 0.7535452618130397,
+      "eval_runtime": 26.824,
+      "eval_samples_per_second": 4.809,
+      "eval_steps_per_second": 1.23,
+      "step": 700
+    },
+    {
+      "epoch": 0.5216426193118757,
+      "grad_norm": 0.92578125,
+      "learning_rate": 1.0993840717432582e-05,
+      "loss": 0.7971,
+      "mean_token_accuracy": 0.7633710459771171,
+      "step": 705
+    },
+    {
+      "epoch": 0.5253422123566408,
+      "grad_norm": 1.0859375,
+      "learning_rate": 1.0865117661582958e-05,
+      "loss": 0.8006,
+      "mean_token_accuracy": 0.7627008957684243,
+      "step": 710
+    },
+    {
+      "epoch": 0.5290418054014059,
+      "grad_norm": 0.90625,
+      "learning_rate": 1.0736250009717249e-05,
+      "loss": 0.8139,
+      "mean_token_accuracy": 0.759724831628023,
+      "step": 715
+    },
+    {
+      "epoch": 0.532741398446171,
+      "grad_norm": 0.84375,
+      "learning_rate": 1.0607259300812047e-05,
+      "loss": 0.7627,
+      "mean_token_accuracy": 0.7734325092398325,
+      "step": 720
+    },
+    {
+      "epoch": 0.536440991490936,
+      "grad_norm": 0.921875,
+      "learning_rate": 1.0478167094411733e-05,
+      "loss": 0.7927,
+      "mean_token_accuracy": 0.7654676870040034,
+      "step": 725
+    },
+    {
+      "epoch": 0.540140584535701,
+      "grad_norm": 0.8203125,
+      "learning_rate": 1.0348994967025012e-05,
+      "loss": 0.7975,
+      "mean_token_accuracy": 0.7640591501043947,
+      "step": 730
+    },
+    {
+      "epoch": 0.5438401775804661,
+      "grad_norm": 0.83203125,
+      "learning_rate": 1.0219764508518595e-05,
+      "loss": 0.7943,
+      "mean_token_accuracy": 0.7650907650226937,
+      "step": 735
+    },
+    {
+      "epoch": 0.5475397706252312,
+      "grad_norm": 0.875,
+      "learning_rate": 1.0090497318508687e-05,
+      "loss": 0.8167,
+      "mean_token_accuracy": 0.7581953026167776,
+      "step": 740
+    },
+    {
+      "epoch": 0.5512393636699963,
+      "grad_norm": 0.84375,
+      "learning_rate": 9.961215002750799e-06,
+      "loss": 0.8268,
+      "mean_token_accuracy": 0.7548056403353621,
+      "step": 745
+    },
+    {
+      "epoch": 0.5549389567147613,
+      "grad_norm": 0.8515625,
+      "learning_rate": 9.831939169528565e-06,
+      "loss": 0.7779,
+      "mean_token_accuracy": 0.7683683284064977,
+      "step": 750
+    },
+    {
+      "epoch": 0.5586385497595264,
+      "grad_norm": 0.9140625,
+      "learning_rate": 9.702691426042124e-06,
+      "loss": 0.7623,
+      "mean_token_accuracy": 0.7725158209911351,
+      "step": 755
+    },
+    {
+      "epoch": 0.5623381428042915,
+      "grad_norm": 0.89453125,
+      "learning_rate": 9.573493374796694e-06,
+      "loss": 0.8002,
+      "mean_token_accuracy": 0.7630235601457683,
+      "step": 760
+    },
+    {
+      "epoch": 0.5660377358490566,
+      "grad_norm": 0.84375,
+      "learning_rate": 9.444366609991916e-06,
+      "loss": 0.8079,
+      "mean_token_accuracy": 0.7589916413466837,
+      "step": 765
+    },
+    {
+      "epoch": 0.5697373288938217,
+      "grad_norm": 0.81640625,
+      "learning_rate": 9.315332713912593e-06,
+      "loss": 0.7346,
+      "mean_token_accuracy": 0.7802107535325717,
+      "step": 770
+    },
+    {
+      "epoch": 0.5734369219385868,
+      "grad_norm": 0.7890625,
+      "learning_rate": 9.18641325332142e-06,
+      "loss": 0.7862,
+      "mean_token_accuracy": 0.7658035573517493,
+      "step": 775
+    },
+    {
+      "epoch": 0.5771365149833518,
+      "grad_norm": 0.80859375,
+      "learning_rate": 9.057629775854314e-06,
+      "loss": 0.7917,
+      "mean_token_accuracy": 0.764880695306027,
+      "step": 780
+    },
+    {
+      "epoch": 0.5808361080281169,
+      "grad_norm": 0.890625,
+      "learning_rate": 8.929003806418934e-06,
+      "loss": 0.7966,
+      "mean_token_accuracy": 0.7636755995055275,
+      "step": 785
+    },
+    {
+      "epoch": 0.584535701072882,
+      "grad_norm": 0.88671875,
+      "learning_rate": 8.800556843597002e-06,
+      "loss": 0.782,
+      "mean_token_accuracy": 0.7669459383984821,
+      "step": 790
+    },
+    {
+      "epoch": 0.5882352941176471,
+      "grad_norm": 0.8125,
+      "learning_rate": 8.672310356051023e-06,
+      "loss": 0.8051,
+      "mean_token_accuracy": 0.7610877406676881,
+      "step": 795
+    },
+    {
+      "epoch": 0.5919348871624122,
+      "grad_norm": 0.81640625,
+      "learning_rate": 8.544285778936004e-06,
+      "loss": 0.7989,
+      "mean_token_accuracy": 0.7634215644199668,
+      "step": 800
+    },
+    {
+      "epoch": 0.5919348871624122,
+      "eval_loss": 0.8252535462379456,
+      "eval_mean_token_accuracy": 0.7536745901089209,
+      "eval_runtime": 26.832,
+      "eval_samples_per_second": 4.808,
+      "eval_steps_per_second": 1.23,
+      "step": 800
+    },
+    {
+      "epoch": 0.5956344802071772,
+      "grad_norm": 0.984375,
+      "learning_rate": 8.416504510316774e-06,
+      "loss": 0.776,
+      "mean_token_accuracy": 0.7695336465462759,
+      "step": 805
+    },
+    {
+      "epoch": 0.5993340732519423,
+      "grad_norm": 0.95703125,
+      "learning_rate": 8.28898790759152e-06,
+      "loss": 0.7969,
+      "mean_token_accuracy": 0.763109347526755,
+      "step": 810
+    },
+    {
+      "epoch": 0.6030336662967074,
+      "grad_norm": 0.921875,
+      "learning_rate": 8.161757283922084e-06,
+      "loss": 0.7782,
+      "mean_token_accuracy": 0.7679017282774507,
+      "step": 815
+    },
+    {
+      "epoch": 0.6067332593414725,
+      "grad_norm": 0.859375,
+      "learning_rate": 8.034833904671698e-06,
+      "loss": 0.7903,
+      "mean_token_accuracy": 0.76566573935166,
+      "step": 820
+    },
+    {
+      "epoch": 0.6104328523862376,
+      "grad_norm": 0.8359375,
+      "learning_rate": 7.908238983850666e-06,
+      "loss": 0.8111,
+      "mean_token_accuracy": 0.7594188487181153,
+      "step": 825
+    },
+    {
+      "epoch": 0.6141324454310025,
+      "grad_norm": 0.83984375,
+      "learning_rate": 7.781993680570656e-06,
+      "loss": 0.8053,
+      "mean_token_accuracy": 0.7628334522803429,
+      "step": 830
+    },
+    {
+      "epoch": 0.6178320384757676,
+      "grad_norm": 0.87890625,
+      "learning_rate": 7.656119095508155e-06,
+      "loss": 0.7773,
+      "mean_token_accuracy": 0.7692651631116968,
+      "step": 835
+    },
+    {
+      "epoch": 0.6215316315205327,
+      "grad_norm": 0.8984375,
+      "learning_rate": 7.530636267377706e-06,
+      "loss": 0.7837,
+      "mean_token_accuracy": 0.7655114768766266,
+      "step": 840
+    },
+    {
+      "epoch": 0.6252312245652978,
+      "grad_norm": 0.83203125,
+      "learning_rate": 7.405566169415481e-06,
+      "loss": 0.8074,
+      "mean_token_accuracy": 0.7589263445679284,
+      "step": 845
+    },
+    {
+      "epoch": 0.6289308176100629,
+      "grad_norm": 0.8984375,
+      "learning_rate": 7.280929705873818e-06,
+      "loss": 0.8338,
+      "mean_token_accuracy": 0.7536498452898096,
+      "step": 850
+    },
+    {
+      "epoch": 0.632630410654828,
+      "grad_norm": 0.94921875,
+      "learning_rate": 7.15674770852727e-06,
+      "loss": 0.8373,
+      "mean_token_accuracy": 0.7523458820439008,
+      "step": 855
+    },
+    {
+      "epoch": 0.636330003699593,
+      "grad_norm": 0.859375,
+      "learning_rate": 7.033040933190776e-06,
+      "loss": 0.8153,
+      "mean_token_accuracy": 0.7589709979146929,
+      "step": 860
+    },
+    {
+      "epoch": 0.6400295967443581,
+      "grad_norm": 0.84375,
+      "learning_rate": 6.909830056250527e-06,
+      "loss": 0.7651,
+      "mean_token_accuracy": 0.772660770269878,
+      "step": 865
+    },
+    {
+      "epoch": 0.6437291897891232,
+      "grad_norm": 0.96484375,
+      "learning_rate": 6.787135671208126e-06,
+      "loss": 0.8238,
+      "mean_token_accuracy": 0.7566547409870067,
+      "step": 870
+    },
+    {
+      "epoch": 0.6474287828338883,
+      "grad_norm": 0.85546875,
+      "learning_rate": 6.6649782852385554e-06,
+      "loss": 0.8101,
+      "mean_token_accuracy": 0.7590977120714553,
+      "step": 875
+    },
+    {
+      "epoch": 0.6511283758786534,
+      "grad_norm": 0.8046875,
+      "learning_rate": 6.543378315762634e-06,
+      "loss": 0.7895,
+      "mean_token_accuracy": 0.7658402945466907,
+      "step": 880
+    },
+    {
+      "epoch": 0.6548279689234184,
+      "grad_norm": 0.84375,
+      "learning_rate": 6.42235608703441e-06,
+      "loss": 0.8261,
+      "mean_token_accuracy": 0.7549317620597372,
+      "step": 885
+    },
+    {
+      "epoch": 0.6585275619681835,
+      "grad_norm": 0.8828125,
+      "learning_rate": 6.301931826744189e-06,
+      "loss": 0.807,
+      "mean_token_accuracy": 0.7609498606703473,
+      "step": 890
+    },
+    {
+      "epoch": 0.6622271550129486,
+      "grad_norm": 0.87890625,
+      "learning_rate": 6.18212566263765e-06,
+      "loss": 0.8069,
+      "mean_token_accuracy": 0.7613288948161152,
+      "step": 895
+    },
+    {
+      "epoch": 0.6659267480577137,
+      "grad_norm": 0.91015625,
+      "learning_rate": 6.0629576191517035e-06,
+      "loss": 0.7901,
+      "mean_token_accuracy": 0.7639538375550893,
+      "step": 900
+    },
+    {
+      "epoch": 0.6659267480577137,
+      "eval_loss": 0.8244156241416931,
+      "eval_mean_token_accuracy": 0.7541191008467915,
+      "eval_runtime": 26.8206,
+      "eval_samples_per_second": 4.81,
+      "eval_steps_per_second": 1.23,
+      "step": 900
+    },
+    {
+      "epoch": 0.6696263411024788,
+      "grad_norm": 0.8359375,
+      "learning_rate": 5.944447614067588e-06,
+      "loss": 0.802,
+      "mean_token_accuracy": 0.7623744418543843,
+      "step": 905
+    },
+    {
+      "epoch": 0.6733259341472438,
+      "grad_norm": 0.87890625,
+      "learning_rate": 5.8266154551818225e-06,
+      "loss": 0.7831,
+      "mean_token_accuracy": 0.7682524160081867,
+      "step": 910
+    },
+    {
+      "epoch": 0.6770255271920089,
+      "grad_norm": 0.90234375,
+      "learning_rate": 5.709480836995509e-06,
+      "loss": 0.7745,
+      "mean_token_accuracy": 0.7695543830336732,
+      "step": 915
+    },
+    {
+      "epoch": 0.680725120236774,
+      "grad_norm": 0.921875,
+      "learning_rate": 5.593063337422595e-06,
+      "loss": 0.7854,
+      "mean_token_accuracy": 0.7675578515782145,
+      "step": 920
+    },
+    {
+      "epoch": 0.684424713281539,
+      "grad_norm": 0.86328125,
+      "learning_rate": 5.477382414517625e-06,
+      "loss": 0.8125,
+      "mean_token_accuracy": 0.7603088368351884,
+      "step": 925
+    },
+    {
+      "epoch": 0.6881243063263041,
+      "grad_norm": 0.86328125,
+      "learning_rate": 5.362457403223495e-06,
+      "loss": 0.7999,
+      "mean_token_accuracy": 0.7644135481602934,
+      "step": 930
+    },
+    {
+      "epoch": 0.6918238993710691,
+      "grad_norm": 0.83203125,
+      "learning_rate": 5.248307512139818e-06,
+      "loss": 0.8023,
+      "mean_token_accuracy": 0.7618850109300808,
+      "step": 935
+    },
+    {
+      "epoch": 0.6955234924158342,
+      "grad_norm": 0.82421875,
+      "learning_rate": 5.134951820312402e-06,
+      "loss": 0.7441,
+      "mean_token_accuracy": 0.7790121524450443,
+      "step": 940
+    },
+    {
+      "epoch": 0.6992230854605993,
+      "grad_norm": 0.80859375,
+      "learning_rate": 5.022409274044346e-06,
+      "loss": 0.8185,
+      "mean_token_accuracy": 0.7567813697048701,
+      "step": 945
+    },
+    {
+      "epoch": 0.7029226785053644,
+      "grad_norm": 0.8125,
+      "learning_rate": 4.910698683729371e-06,
+      "loss": 0.7907,
+      "mean_token_accuracy": 0.7658204482281679,
+      "step": 950
+    },
+    {
+      "epoch": 0.7066222715501295,
+      "grad_norm": 0.8828125,
+      "learning_rate": 4.799838720707847e-06,
+      "loss": 0.7396,
+      "mean_token_accuracy": 0.7788681798179564,
+      "step": 955
+    },
+    {
+      "epoch": 0.7103218645948945,
+      "grad_norm": 0.953125,
+      "learning_rate": 4.6898479141460415e-06,
+      "loss": 0.7685,
+      "mean_token_accuracy": 0.7716848343018439,
+      "step": 960
+    },
+    {
+      "epoch": 0.7140214576396596,
+      "grad_norm": 0.890625,
+      "learning_rate": 4.580744647939163e-06,
+      "loss": 0.8112,
+      "mean_token_accuracy": 0.7598053624319068,
+      "step": 965
+    },
+    {
+      "epoch": 0.7177210506844247,
+      "grad_norm": 0.91015625,
+      "learning_rate": 4.472547157638674e-06,
+      "loss": 0.7594,
+      "mean_token_accuracy": 0.7750113751923063,
+      "step": 970
+    },
+    {
+      "epoch": 0.7214206437291898,
+      "grad_norm": 0.91796875,
+      "learning_rate": 4.365273527404384e-06,
+      "loss": 0.808,
+      "mean_token_accuracy": 0.7603356237743368,
+      "step": 975
+    },
+    {
+      "epoch": 0.7251202367739549,
+      "grad_norm": 0.87109375,
+      "learning_rate": 4.258941686981864e-06,
+      "loss": 0.7955,
+      "mean_token_accuracy": 0.7642262507650953,
+      "step": 980
+    },
+    {
+      "epoch": 0.72881982981872,
+      "grad_norm": 1.0546875,
+      "learning_rate": 4.15356940870567e-06,
+      "loss": 0.8068,
+      "mean_token_accuracy": 0.7613863104054104,
+      "step": 985
+    },
+    {
+      "epoch": 0.732519422863485,
+      "grad_norm": 0.90625,
+      "learning_rate": 4.049174304528857e-06,
+      "loss": 0.7822,
+      "mean_token_accuracy": 0.7685320827285723,
+      "step": 990
+    },
+    {
+      "epoch": 0.7362190159082501,
+      "grad_norm": 0.87109375,
+      "learning_rate": 3.945773823079315e-06,
+      "loss": 0.7731,
+      "mean_token_accuracy": 0.7706841039817524,
+      "step": 995
+    },
+    {
+      "epoch": 0.7399186089530152,
+      "grad_norm": 0.87109375,
+      "learning_rate": 3.8433852467434175e-06,
+      "loss": 0.8149,
+      "mean_token_accuracy": 0.7571042333201665,
+      "step": 1000
+    },
+    {
+      "epoch": 0.7399186089530152,
+      "eval_loss": 0.8240998387336731,
+      "eval_mean_token_accuracy": 0.754324827933146,
+      "eval_runtime": 26.8419,
+      "eval_samples_per_second": 4.806,
+      "eval_steps_per_second": 1.229,
+      "step": 1000
+    },
+    {
+      "epoch": 0.7436182019977803,
+      "grad_norm": 1.109375,
+      "learning_rate": 3.742025688777413e-06,
+      "loss": 0.8229,
+      "mean_token_accuracy": 0.7550693105368848,
+      "step": 1005
+    },
+    {
+      "epoch": 0.7473177950425454,
+      "grad_norm": 0.87109375,
+      "learning_rate": 3.641712090447125e-06,
+      "loss": 0.7788,
+      "mean_token_accuracy": 0.7685615689165098,
+      "step": 1010
+    },
+    {
+      "epoch": 0.7510173880873104,
+      "grad_norm": 0.8359375,
+      "learning_rate": 3.542461218196379e-06,
+      "loss": 0.788,
+      "mean_token_accuracy": 0.7655549040795401,
+      "step": 1015
+    },
+    {
+      "epoch": 0.7547169811320755,
+      "grad_norm": 1.046875,
+      "learning_rate": 3.444289660844665e-06,
+      "loss": 0.8077,
+      "mean_token_accuracy": 0.761921876795666,
+      "step": 1020
+    },
+    {
+      "epoch": 0.7584165741768405,
+      "grad_norm": 0.83984375,
+      "learning_rate": 3.347213826814456e-06,
+      "loss": 0.8061,
+      "mean_token_accuracy": 0.7599252603977095,
+      "step": 1025
+    },
+    {
+      "epoch": 0.7621161672216056,
+      "grad_norm": 0.90234375,
+      "learning_rate": 3.2512499413887255e-06,
+      "loss": 0.7914,
+      "mean_token_accuracy": 0.7653928014723982,
+      "step": 1030
+    },
+    {
+      "epoch": 0.7658157602663707,
+      "grad_norm": 0.875,
+      "learning_rate": 3.1564140439990256e-06,
+      "loss": 0.8263,
+      "mean_token_accuracy": 0.7550946774073029,
+      "step": 1035
+    },
+    {
+      "epoch": 0.7695153533111357,
+      "grad_norm": 0.84375,
+      "learning_rate": 3.0627219855446667e-06,
+      "loss": 0.761,
+      "mean_token_accuracy": 0.7739465199591922,
+      "step": 1040
+    },
+    {
+      "epoch": 0.7732149463559008,
+      "grad_norm": 0.8359375,
+      "learning_rate": 2.970189425743383e-06,
+      "loss": 0.8089,
+      "mean_token_accuracy": 0.759677717877101,
+      "step": 1045
+    },
+    {
+      "epoch": 0.7769145394006659,
+      "grad_norm": 0.8203125,
+      "learning_rate": 2.8788318305139808e-06,
+      "loss": 0.824,
+      "mean_token_accuracy": 0.7545435539401769,
+      "step": 1050
+    },
+    {
+      "epoch": 0.780614132445431,
+      "grad_norm": 0.84375,
+      "learning_rate": 2.7886644693913333e-06,
+      "loss": 0.8149,
+      "mean_token_accuracy": 0.7587312870989174,
+      "step": 1055
+    },
+    {
+      "epoch": 0.7843137254901961,
+      "grad_norm": 0.9453125,
+      "learning_rate": 2.6997024129742544e-06,
+      "loss": 0.8031,
+      "mean_token_accuracy": 0.7626208873426253,
+      "step": 1060
+    },
+    {
+      "epoch": 0.7880133185349611,
+      "grad_norm": 0.875,
+      "learning_rate": 2.611960530406572e-06,
+      "loss": 0.7958,
+      "mean_token_accuracy": 0.7655382203600427,
+      "step": 1065
+    },
+    {
+      "epoch": 0.7917129115797262,
+      "grad_norm": 0.86328125,
+      "learning_rate": 2.5254534868919077e-06,
+      "loss": 0.7976,
+      "mean_token_accuracy": 0.7625642551460374,
+      "step": 1070
+    },
+    {
+      "epoch": 0.7954125046244913,
+      "grad_norm": 0.82421875,
+      "learning_rate": 2.4401957412425213e-06,
+      "loss": 0.804,
+      "mean_token_accuracy": 0.7610083478042525,
+      "step": 1075
+    },
+    {
+      "epoch": 0.7991120976692564,
+      "grad_norm": 0.87890625,
+      "learning_rate": 2.3562015434626784e-06,
+      "loss": 0.7854,
+      "mean_token_accuracy": 0.7672943500462276,
+      "step": 1080
+    },
+    {
+      "epoch": 0.8028116907140215,
+      "grad_norm": 0.84375,
+      "learning_rate": 2.273484932366874e-06,
+      "loss": 0.7757,
+      "mean_token_accuracy": 0.7699604458649688,
+      "step": 1085
+    },
+    {
+      "epoch": 0.8065112837587866,
+      "grad_norm": 0.83984375,
+      "learning_rate": 2.192059733233408e-06,
+      "loss": 0.7937,
+      "mean_token_accuracy": 0.7643169173242313,
+      "step": 1090
+    },
+    {
+      "epoch": 0.8102108768035516,
+      "grad_norm": 0.87109375,
+      "learning_rate": 2.111939555493603e-06,
+      "loss": 0.795,
+      "mean_token_accuracy": 0.7636067259486805,
+      "step": 1095
+    },
+    {
+      "epoch": 0.8139104698483167,
+      "grad_norm": 0.95703125,
+      "learning_rate": 2.0331377904571303e-06,
+      "loss": 0.8168,
+      "mean_token_accuracy": 0.7588089596935865,
+      "step": 1100
+    },
+    {
+      "epoch": 0.8139104698483167,
+      "eval_loss": 0.8239597082138062,
+      "eval_mean_token_accuracy": 0.7540469434367087,
+      "eval_runtime": 26.8406,
+      "eval_samples_per_second": 4.806,
+      "eval_steps_per_second": 1.229,
+      "step": 1100
+    },
+    {
+      "epoch": 0.8176100628930818,
+      "grad_norm": 0.87109375,
+      "learning_rate": 1.9556676090737803e-06,
+      "loss": 0.8288,
+      "mean_token_accuracy": 0.7553538104703874,
+      "step": 1105
+    },
+    {
+      "epoch": 0.8213096559378469,
+      "grad_norm": 0.8515625,
+      "learning_rate": 1.879541959732072e-06,
+      "loss": 0.7833,
+      "mean_token_accuracy": 0.7671695181788402,
+      "step": 1110
+    },
+    {
+      "epoch": 0.825009248982612,
+      "grad_norm": 0.9296875,
+      "learning_rate": 1.8047735660950427e-06,
+      "loss": 0.7812,
+      "mean_token_accuracy": 0.769421476075401,
+      "step": 1115
+    },
+    {
+      "epoch": 0.8287088420273769,
+      "grad_norm": 0.875,
+      "learning_rate": 1.7313749249736266e-06,
+      "loss": 0.7926,
+      "mean_token_accuracy": 0.7657877450561917,
+      "step": 1120
+    },
+    {
+      "epoch": 0.832408435072142,
+      "grad_norm": 0.890625,
+      "learning_rate": 1.6593583042379192e-06,
+      "loss": 0.8069,
+      "mean_token_accuracy": 0.7609272634496878,
+      "step": 1125
+    },
+    {
+      "epoch": 0.8361080281169071,
+      "grad_norm": 0.97265625,
+      "learning_rate": 1.5887357407667314e-06,
+      "loss": 0.809,
+      "mean_token_accuracy": 0.7607928652377135,
+      "step": 1130
+    },
+    {
+      "epoch": 0.8398076211616722,
+      "grad_norm": 0.90625,
+      "learning_rate": 1.5195190384357405e-06,
+      "loss": 0.7972,
+      "mean_token_accuracy": 0.7646063519791061,
+      "step": 1135
+    },
+    {
+      "epoch": 0.8435072142064373,
+      "grad_norm": 0.80859375,
+      "learning_rate": 1.4517197661445893e-06,
+      "loss": 0.7793,
+      "mean_token_accuracy": 0.7676166182089534,
+      "step": 1140
+    },
+    {
+      "epoch": 0.8472068072512023,
+      "grad_norm": 0.8203125,
+      "learning_rate": 1.3853492558832472e-06,
+      "loss": 0.7983,
+      "mean_token_accuracy": 0.7633139122401321,
+      "step": 1145
+    },
+    {
+      "epoch": 0.8509064002959674,
+      "grad_norm": 0.8984375,
+      "learning_rate": 1.3204186008379926e-06,
+      "loss": 0.7873,
+      "mean_token_accuracy": 0.7678782526512846,
+      "step": 1150
+    },
+    {
+      "epoch": 0.8546059933407325,
+      "grad_norm": 0.875,
+      "learning_rate": 1.2569386535372807e-06,
+      "loss": 0.8111,
+      "mean_token_accuracy": 0.7590790517116295,
+      "step": 1155
+    },
+    {
+      "epoch": 0.8583055863854976,
+      "grad_norm": 0.9921875,
+      "learning_rate": 1.1949200240378577e-06,
+      "loss": 0.7794,
+      "mean_token_accuracy": 0.7682778315162649,
+      "step": 1160
+    },
+    {
+      "epoch": 0.8620051794302627,
+      "grad_norm": 0.85546875,
+      "learning_rate": 1.1343730781513896e-06,
+      "loss": 0.7822,
+      "mean_token_accuracy": 0.7676008093864439,
+      "step": 1165
+    },
+    {
+      "epoch": 0.8657047724750278,
+      "grad_norm": 0.80859375,
+      "learning_rate": 1.0753079357119134e-06,
+      "loss": 0.8067,
+      "mean_token_accuracy": 0.7606890149530049,
+      "step": 1170
+    },
+    {
+      "epoch": 0.8694043655197928,
+      "grad_norm": 0.875,
+      "learning_rate": 1.017734468884417e-06,
+      "loss": 0.7573,
+      "mean_token_accuracy": 0.7753776235476382,
+      "step": 1175
+    },
+    {
+      "epoch": 0.8731039585645579,
+      "grad_norm": 0.859375,
+      "learning_rate": 9.616623005147952e-07,
+      "loss": 0.8182,
+      "mean_token_accuracy": 0.7570817494844897,
+      "step": 1180
+    },
+    {
+      "epoch": 0.876803551609323,
+      "grad_norm": 0.859375,
+      "learning_rate": 9.071008025214767e-07,
+      "loss": 0.7372,
+      "mean_token_accuracy": 0.7811912064424342,
+      "step": 1185
+    },
+    {
+      "epoch": 0.8805031446540881,
+      "grad_norm": 0.81640625,
+      "learning_rate": 8.540590943290128e-07,
+      "loss": 0.811,
+      "mean_token_accuracy": 0.7593687259772598,
+      "step": 1190
+    },
+    {
+      "epoch": 0.8842027376988532,
+      "grad_norm": 0.80859375,
+      "learning_rate": 8.025460413438457e-07,
+      "loss": 0.7864,
+      "mean_token_accuracy": 0.7669303318911965,
+      "step": 1195
+    },
+    {
+      "epoch": 0.8879023307436182,
+      "grad_norm": 0.8203125,
+      "learning_rate": 7.525702534725443e-07,
+      "loss": 0.7953,
+      "mean_token_accuracy": 0.7641527053640121,
+      "step": 1200
+    },
+    {
+      "epoch": 0.8879023307436182,
+      "eval_loss": 0.8239200711250305,
+      "eval_mean_token_accuracy": 0.7543525912020813,
+      "eval_runtime": 26.8235,
+      "eval_samples_per_second": 4.809,
+      "eval_steps_per_second": 1.23,
+      "step": 1200
+    },
+    {
+      "epoch": 0.8916019237883833,
+      "grad_norm": 0.87109375,
+      "learning_rate": 7.041400836827439e-07,
+      "loss": 0.7879,
+      "mean_token_accuracy": 0.7669332209292096,
+      "step": 1205
+    },
+    {
+      "epoch": 0.8953015168331484,
+      "grad_norm": 0.859375,
+      "learning_rate": 6.572636266070265e-07,
+      "loss": 0.8129,
+      "mean_token_accuracy": 0.7596671779447364,
+      "step": 1210
+    },
+    {
+      "epoch": 0.8990011098779135,
+      "grad_norm": 0.86328125,
+      "learning_rate": 6.119487171899807e-07,
+      "loss": 0.765,
+      "mean_token_accuracy": 0.7727977613654252,
+      "step": 1215
+    },
+    {
+      "epoch": 0.9027007029226785,
+      "grad_norm": 0.93359375,
+      "learning_rate": 5.682029293786673e-07,
+      "loss": 0.7967,
+      "mean_token_accuracy": 0.7628338654986015,
+      "step": 1220
+    },
+    {
+      "epoch": 0.9064002959674435,
+      "grad_norm": 0.8046875,
+      "learning_rate": 5.26033574856708e-07,
+      "loss": 0.7769,
+      "mean_token_accuracy": 0.7688456068733477,
+      "step": 1225
+    },
+    {
+      "epoch": 0.9100998890122086,
+      "grad_norm": 0.80078125,
+      "learning_rate": 4.854477018222103e-07,
+      "loss": 0.7812,
+      "mean_token_accuracy": 0.7679284651491448,
+      "step": 1230
+    },
+    {
+      "epoch": 0.9137994820569737,
+      "grad_norm": 0.85546875,
+      "learning_rate": 4.464520938097294e-07,
+      "loss": 0.7243,
+      "mean_token_accuracy": 0.7839677244091058,
+      "step": 1235
+    },
+    {
+      "epoch": 0.9174990751017388,
+      "grad_norm": 0.91796875,
+      "learning_rate": 4.0905326855646186e-07,
+      "loss": 0.7739,
+      "mean_token_accuracy": 0.7714798694155494,
+      "step": 1240
+    },
+    {
+      "epoch": 0.9211986681465039,
+      "grad_norm": 0.9140625,
+      "learning_rate": 3.732574769128738e-07,
+      "loss": 0.8152,
+      "mean_token_accuracy": 0.7584452298165504,
+      "step": 1245
+    },
+    {
+      "epoch": 0.9248982611912689,
+      "grad_norm": 0.9296875,
+      "learning_rate": 3.390707017979311e-07,
+      "loss": 0.7847,
+      "mean_token_accuracy": 0.7676911980262516,
+      "step": 1250
+    },
+    {
+      "epoch": 0.928597854236034,
+      "grad_norm": 0.90234375,
+      "learning_rate": 3.06498657199108e-07,
+      "loss": 0.7661,
+      "mean_token_accuracy": 0.7724968048968479,
+      "step": 1255
+    },
+    {
+      "epoch": 0.9322974472807991,
+      "grad_norm": 0.984375,
+      "learning_rate": 2.7554678721735675e-07,
+      "loss": 0.7977,
+      "mean_token_accuracy": 0.7636390963976949,
+      "step": 1260
+    },
+    {
+      "epoch": 0.9359970403255642,
+      "grad_norm": 0.828125,
+      "learning_rate": 2.4622026515717654e-07,
+      "loss": 0.7834,
+      "mean_token_accuracy": 0.7677288523241763,
+      "step": 1265
+    },
+    {
+      "epoch": 0.9396966333703293,
+      "grad_norm": 0.84765625,
+      "learning_rate": 2.1852399266194312e-07,
+      "loss": 0.7772,
+      "mean_token_accuracy": 0.768929287333916,
+      "step": 1270
+    },
+    {
+      "epoch": 0.9433962264150944,
+      "grad_norm": 0.99609375,
+      "learning_rate": 1.9246259889464935e-07,
+      "loss": 0.8122,
+      "mean_token_accuracy": 0.7599383528366281,
+      "step": 1275
+    },
+    {
+      "epoch": 0.9470958194598594,
+      "grad_norm": 0.890625,
+      "learning_rate": 1.6804043976418438e-07,
+      "loss": 0.7858,
+      "mean_token_accuracy": 0.7668953904069555,
+      "step": 1280
+    },
+    {
+      "epoch": 0.9507954125046245,
+      "grad_norm": 0.8359375,
+      "learning_rate": 1.4526159719728595e-07,
+      "loss": 0.7604,
+      "mean_token_accuracy": 0.7734440775669997,
+      "step": 1285
+    },
+    {
+      "epoch": 0.9544950055493896,
+      "grad_norm": 0.8984375,
+      "learning_rate": 1.24129878456285e-07,
+      "loss": 0.798,
+      "mean_token_accuracy": 0.7641898244956924,
+      "step": 1290
+    },
+    {
+      "epoch": 0.9581945985941547,
+      "grad_norm": 0.8828125,
+      "learning_rate": 1.0464881550276362e-07,
+      "loss": 0.7848,
+      "mean_token_accuracy": 0.7662787461098154,
+      "step": 1295
+    },
+    {
+      "epoch": 0.9618941916389198,
+      "grad_norm": 0.875,
+      "learning_rate": 8.682166440721729e-08,
+      "loss": 0.7568,
+      "mean_token_accuracy": 0.7745805700591353,
+      "step": 1300
+    },
+    {
+      "epoch": 0.9618941916389198,
+      "eval_loss": 0.8239119052886963,
+      "eval_mean_token_accuracy": 0.7541335920647368,
+      "eval_runtime": 26.8223,
+      "eval_samples_per_second": 4.809,
+      "eval_steps_per_second": 1.23,
+      "step": 1300
+    },
+    {
+      "epoch": 0.9655937846836848,
+      "grad_norm": 0.80078125,
+      "learning_rate": 7.065140480483235e-08,
+      "loss": 0.7962,
+      "mean_token_accuracy": 0.7651817692524249,
+      "step": 1305
+    },
+    {
+      "epoch": 0.9692933777284499,
+      "grad_norm": 0.8359375,
+      "learning_rate": 5.6140739397474445e-08,
+      "loss": 0.8162,
+      "mean_token_accuracy": 0.7582110748345693,
+      "step": 1310
+    },
+    {
+      "epoch": 0.9729929707732149,
+      "grad_norm": 0.8671875,
+      "learning_rate": 4.329209350195651e-08,
+      "loss": 0.804,
+      "mean_token_accuracy": 0.7607380437494441,
+      "step": 1315
+    },
+    {
+      "epoch": 0.97669256381798,
+      "grad_norm": 0.81640625,
+      "learning_rate": 3.210761464466639e-08,
+      "loss": 0.749,
+      "mean_token_accuracy": 0.7764496700263599,
+      "step": 1320
+    },
+    {
+      "epoch": 0.9803921568627451,
+      "grad_norm": 0.9609375,
+      "learning_rate": 2.2589172202635014e-08,
+      "loss": 0.7914,
+      "mean_token_accuracy": 0.7662956108439277,
+      "step": 1325
+    },
+    {
+      "epoch": 0.9840917499075101,
+      "grad_norm": 0.99609375,
+      "learning_rate": 1.4738357091084177e-08,
+      "loss": 0.7969,
+      "mean_token_accuracy": 0.7634388673702442,
+      "step": 1330
+    },
+    {
+      "epoch": 0.9877913429522752,
+      "grad_norm": 0.8984375,
+      "learning_rate": 8.556481497521418e-09,
+      "loss": 0.7807,
+      "mean_token_accuracy": 0.766738749692242,
+      "step": 1335
+    },
+    {
+      "epoch": 0.9914909359970403,
+      "grad_norm": 0.90234375,
+      "learning_rate": 4.044578662419918e-09,
+      "loss": 0.7919,
+      "mean_token_accuracy": 0.7654007144712723,
+      "step": 1340
+    },
+    {
+      "epoch": 0.9951905290418054,
+      "grad_norm": 0.87890625,
+      "learning_rate": 1.203402706525525e-09,
+      "loss": 0.8252,
+      "mean_token_accuracy": 0.7551888034019024,
+      "step": 1345
+    },
+    {
+      "epoch": 0.9988901220865705,
+      "grad_norm": 0.90625,
+      "learning_rate": 3.342850480869686e-11,
+      "loss": 0.8057,
+      "mean_token_accuracy": 0.7610222629972448,
+      "step": 1350
+    },
+    {
+      "epoch": 0.9996300406955235,
+      "mean_token_accuracy": 0.7605873625040888,
+      "step": 1351,
+      "total_flos": 6.960985517882081e+17,
+      "train_loss": 0.8210960451450108,
+      "train_runtime": 16213.9925,
+      "train_samples_per_second": 1.334,
+      "train_steps_per_second": 0.083
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1351,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 6.960985517882081e+17,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a8c65df2911bd1ab2de2ffd424c02b43a7427770c945bf29822ae3f29add721
+size 6008