Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,209 @@
+---
+base_model: Qwen/Qwen3-1.7B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:Qwen/Qwen3-1.7B
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.17.1

adapter_config.json

@@ -0,0 +1,42 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen3-1.7B",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.01,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "qalora_group_size": 16,
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "up_proj",
+    "v_proj",
+    "down_proj",
+    "gate_proj",
+    "q_proj",
+    "k_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:209c54baa7d30c4f45491b46b2f32ec97ccf36127d0a75fa39b068d707cb25a9
+size 278973888

added_tokens.json

@@ -0,0 +1,28 @@
+{
+  "</think>": 151668,
+  "</tool_call>": 151658,
+  "</tool_response>": 151666,
+  "<think>": 151667,
+  "<tool_call>": 151657,
+  "<tool_response>": 151665,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

chat_template.jinja

@@ -0,0 +1,89 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f569614b6083fb1bbd1f3cce092fd82355b341cc2e517868f5122b14d9af123
+size 558179131

rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d8b782f88586f3c2586e1c987178e4600b1344b70f5e93a8fea25ee3c25da9e
+size 14645

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b45a3a293ebfc3febdd6cddb01f348c255e798679549379734d47a97ae6c756
+size 1465

special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4
+size 11422654

tokenizer_config.json

@@ -0,0 +1,239 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151666": {
+      "content": "</tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151667": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151668": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

trainer_state.json

@@ -0,0 +1,2634 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.625871611269191,
+  "eval_steps": 500,
+  "global_step": 13000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.57754989027977,
+      "epoch": 0.006253713142178169,
+      "grad_norm": 0.1328946352005005,
+      "learning_rate": 8.166666666666667e-05,
+      "loss": 1.6654,
+      "mean_token_accuracy": 0.5821028184890747,
+      "num_tokens": 1453934.0,
+      "step": 50
+    },
+    {
+      "entropy": 1.7420939958095552,
+      "epoch": 0.012507426284356337,
+      "grad_norm": 0.11827201396226883,
+      "learning_rate": 0.000165,
+      "loss": 1.3586,
+      "mean_token_accuracy": 0.6275478279590607,
+      "num_tokens": 2912735.0,
+      "step": 100
+    },
+    {
+      "entropy": 1.5568377447128297,
+      "epoch": 0.018761139426534505,
+      "grad_norm": 0.15163880586624146,
+      "learning_rate": 0.0002483333333333333,
+      "loss": 1.3207,
+      "mean_token_accuracy": 0.6348841637372971,
+      "num_tokens": 4361799.0,
+      "step": 150
+    },
+    {
+      "entropy": 1.4739533269405365,
+      "epoch": 0.025014852568712675,
+      "grad_norm": 0.12787143886089325,
+      "learning_rate": 0.0003316666666666667,
+      "loss": 1.2842,
+      "mean_token_accuracy": 0.6424031159281731,
+      "num_tokens": 5814731.0,
+      "step": 200
+    },
+    {
+      "entropy": 1.4483512490987778,
+      "epoch": 0.031268565710890844,
+      "grad_norm": 0.1128445640206337,
+      "learning_rate": 0.000415,
+      "loss": 1.2705,
+      "mean_token_accuracy": 0.6454238125681877,
+      "num_tokens": 7268127.0,
+      "step": 250
+    },
+    {
+      "entropy": 1.4052385699748993,
+      "epoch": 0.03752227885306901,
+      "grad_norm": 0.11171138286590576,
+      "learning_rate": 0.0004983333333333334,
+      "loss": 1.2476,
+      "mean_token_accuracy": 0.6496384194493294,
+      "num_tokens": 8718544.0,
+      "step": 300
+    },
+    {
+      "entropy": 1.4152175986766815,
+      "epoch": 0.04377599199524718,
+      "grad_norm": 0.11977560073137283,
+      "learning_rate": 0.0005816666666666666,
+      "loss": 1.2513,
+      "mean_token_accuracy": 0.6493740159273148,
+      "num_tokens": 10172719.0,
+      "step": 350
+    },
+    {
+      "entropy": 1.4048930382728577,
+      "epoch": 0.05002970513742535,
+      "grad_norm": 0.1243417039513588,
+      "learning_rate": 0.000665,
+      "loss": 1.2289,
+      "mean_token_accuracy": 0.6540437403321266,
+      "num_tokens": 11620279.0,
+      "step": 400
+    },
+    {
+      "entropy": 1.4123985522985458,
+      "epoch": 0.056283418279603516,
+      "grad_norm": 0.11343871802091599,
+      "learning_rate": 0.0007483333333333333,
+      "loss": 1.239,
+      "mean_token_accuracy": 0.6520217537879944,
+      "num_tokens": 13073226.0,
+      "step": 450
+    },
+    {
+      "entropy": 1.4004585057497025,
+      "epoch": 0.06253713142178169,
+      "grad_norm": 0.11357053369283676,
+      "learning_rate": 0.0008316666666666666,
+      "loss": 1.225,
+      "mean_token_accuracy": 0.6544527187943459,
+      "num_tokens": 14533915.0,
+      "step": 500
+    },
+    {
+      "entropy": 1.4031615656614305,
+      "epoch": 0.06879084456395985,
+      "grad_norm": 0.12513332068920135,
+      "learning_rate": 0.000915,
+      "loss": 1.2343,
+      "mean_token_accuracy": 0.6527539578080177,
+      "num_tokens": 15980014.0,
+      "step": 550
+    },
+    {
+      "entropy": 1.4137125009298324,
+      "epoch": 0.07504455770613802,
+      "grad_norm": 0.13136321306228638,
+      "learning_rate": 0.0009983333333333333,
+      "loss": 1.2207,
+      "mean_token_accuracy": 0.6558423298597336,
+      "num_tokens": 17432166.0,
+      "step": 600
+    },
+    {
+      "entropy": 1.4449491912126542,
+      "epoch": 0.08129827084831619,
+      "grad_norm": 0.1370031088590622,
+      "learning_rate": 0.0010816666666666668,
+      "loss": 1.239,
+      "mean_token_accuracy": 0.6518874534964562,
+      "num_tokens": 18882117.0,
+      "step": 650
+    },
+    {
+      "entropy": 1.4782390099763871,
+      "epoch": 0.08755198399049435,
+      "grad_norm": 0.15503117442131042,
+      "learning_rate": 0.001165,
+      "loss": 1.244,
+      "mean_token_accuracy": 0.6515460327267647,
+      "num_tokens": 20330658.0,
+      "step": 700
+    },
+    {
+      "entropy": 1.4976489871740342,
+      "epoch": 0.09380569713267252,
+      "grad_norm": 0.14647236466407776,
+      "learning_rate": 0.0012483333333333333,
+      "loss": 1.2517,
+      "mean_token_accuracy": 0.6499434402585029,
+      "num_tokens": 21786059.0,
+      "step": 750
+    },
+    {
+      "entropy": 1.503735830783844,
+      "epoch": 0.1000594102748507,
+      "grad_norm": 0.16612057387828827,
+      "learning_rate": 0.0013316666666666668,
+      "loss": 1.2497,
+      "mean_token_accuracy": 0.6501839146018028,
+      "num_tokens": 23240083.0,
+      "step": 800
+    },
+    {
+      "entropy": 1.5280751663446426,
+      "epoch": 0.10631312341702887,
+      "grad_norm": 0.17178034782409668,
+      "learning_rate": 0.001415,
+      "loss": 1.2609,
+      "mean_token_accuracy": 0.6481665202975273,
+      "num_tokens": 24700708.0,
+      "step": 850
+    },
+    {
+      "entropy": 1.5576636075973511,
+      "epoch": 0.11256683655920703,
+      "grad_norm": 0.18341563642024994,
+      "learning_rate": 0.0014983333333333333,
+      "loss": 1.2956,
+      "mean_token_accuracy": 0.6411516669392586,
+      "num_tokens": 26155197.0,
+      "step": 900
+    },
+    {
+      "entropy": 1.577981666326523,
+      "epoch": 0.1188205497013852,
+      "grad_norm": 0.18315236270427704,
+      "learning_rate": 0.0015816666666666668,
+      "loss": 1.2857,
+      "mean_token_accuracy": 0.643715631365776,
+      "num_tokens": 27601258.0,
+      "step": 950
+    },
+    {
+      "entropy": 1.60272319316864,
+      "epoch": 0.12507426284356338,
+      "grad_norm": 0.19144219160079956,
+      "learning_rate": 0.001665,
+      "loss": 1.3031,
+      "mean_token_accuracy": 0.6408327847719193,
+      "num_tokens": 29063691.0,
+      "step": 1000
+    },
+    {
+      "entropy": 1.5899927777051925,
+      "epoch": 0.13132797598574153,
+      "grad_norm": 0.21011324226856232,
+      "learning_rate": 0.0017483333333333333,
+      "loss": 1.3131,
+      "mean_token_accuracy": 0.6375964027643204,
+      "num_tokens": 30519258.0,
+      "step": 1050
+    },
+    {
+      "entropy": 1.611317069530487,
+      "epoch": 0.1375816891279197,
+      "grad_norm": 0.20436125993728638,
+      "learning_rate": 0.0018316666666666668,
+      "loss": 1.3282,
+      "mean_token_accuracy": 0.6352240258455276,
+      "num_tokens": 31982975.0,
+      "step": 1100
+    },
+    {
+      "entropy": 1.652041432261467,
+      "epoch": 0.14383540227009786,
+      "grad_norm": 0.21256397664546967,
+      "learning_rate": 0.001915,
+      "loss": 1.34,
+      "mean_token_accuracy": 0.6334743791818619,
+      "num_tokens": 33435147.0,
+      "step": 1150
+    },
+    {
+      "entropy": 1.7145874083042145,
+      "epoch": 0.15008911541227604,
+      "grad_norm": 0.2906016409397125,
+      "learning_rate": 0.0019983333333333333,
+      "loss": 1.3717,
+      "mean_token_accuracy": 0.6267069867253303,
+      "num_tokens": 34890160.0,
+      "step": 1200
+    },
+    {
+      "entropy": 1.7395102292299272,
+      "epoch": 0.1563428285544542,
+      "grad_norm": 0.2289716899394989,
+      "learning_rate": 0.0019999771835624504,
+      "loss": 1.3875,
+      "mean_token_accuracy": 0.6242620906233788,
+      "num_tokens": 36342500.0,
+      "step": 1250
+    },
+    {
+      "entropy": 1.7384645998477937,
+      "epoch": 0.16259654169663237,
+      "grad_norm": 0.2350999265909195,
+      "learning_rate": 0.0019999068632721773,
+      "loss": 1.3988,
+      "mean_token_accuracy": 0.6232203379273414,
+      "num_tokens": 37799801.0,
+      "step": 1300
+    },
+    {
+      "entropy": 1.7248098158836365,
+      "epoch": 0.16885025483881055,
+      "grad_norm": 0.2444753348827362,
+      "learning_rate": 0.0019997890329674856,
+      "loss": 1.4063,
+      "mean_token_accuracy": 0.6213877013325692,
+      "num_tokens": 39261919.0,
+      "step": 1350
+    },
+    {
+      "entropy": 1.7469890892505646,
+      "epoch": 0.1751039679809887,
+      "grad_norm": 0.2528238296508789,
+      "learning_rate": 0.001999623698247015,
+      "loss": 1.4028,
+      "mean_token_accuracy": 0.6217664569616318,
+      "num_tokens": 40711313.0,
+      "step": 1400
+    },
+    {
+      "entropy": 1.7513956505060195,
+      "epoch": 0.1813576811231669,
+      "grad_norm": 0.23596389591693878,
+      "learning_rate": 0.001999410866966553,
+      "loss": 1.4098,
+      "mean_token_accuracy": 0.6196629667282104,
+      "num_tokens": 42168864.0,
+      "step": 1450
+    },
+    {
+      "entropy": 1.7314427828788757,
+      "epoch": 0.18761139426534504,
+      "grad_norm": 0.24437537789344788,
+      "learning_rate": 0.001999150549238658,
+      "loss": 1.411,
+      "mean_token_accuracy": 0.6204391068220139,
+      "num_tokens": 43627161.0,
+      "step": 1500
+    },
+    {
+      "entropy": 1.7699194413423538,
+      "epoch": 0.19386510740752322,
+      "grad_norm": 0.24354827404022217,
+      "learning_rate": 0.0019988427574321815,
+      "loss": 1.4275,
+      "mean_token_accuracy": 0.6180677771568298,
+      "num_tokens": 45082614.0,
+      "step": 1550
+    },
+    {
+      "entropy": 3.1429525250196457,
+      "epoch": 0.2001188205497014,
+      "grad_norm": 26.936569213867188,
+      "learning_rate": 0.001998487506171678,
+      "loss": 4.3017,
+      "mean_token_accuracy": 0.44566510522061437,
+      "num_tokens": 46532177.0,
+      "step": 1600
+    },
+    {
+      "entropy": 6.903752332925796,
+      "epoch": 0.20637253369187955,
+      "grad_norm": 0.43571025133132935,
+      "learning_rate": 0.001998084812336713,
+      "loss": 8.4816,
+      "mean_token_accuracy": 0.03934380377701018,
+      "num_tokens": 47987350.0,
+      "step": 1650
+    },
+    {
+      "entropy": 7.173221395015717,
+      "epoch": 0.21262624683405773,
+      "grad_norm": 0.3482828438282013,
+      "learning_rate": 0.001997634695061058,
+      "loss": 7.2052,
+      "mean_token_accuracy": 0.06305525604635477,
+      "num_tokens": 49446129.0,
+      "step": 1700
+    },
+    {
+      "entropy": 6.704223630428314,
+      "epoch": 0.21887995997623588,
+      "grad_norm": 0.25398287177085876,
+      "learning_rate": 0.0019971371757317832,
+      "loss": 6.7459,
+      "mean_token_accuracy": 0.08217163875699043,
+      "num_tokens": 50893739.0,
+      "step": 1750
+    },
+    {
+      "entropy": 6.309946813583374,
+      "epoch": 0.22513367311841406,
+      "grad_norm": 0.27875518798828125,
+      "learning_rate": 0.001996592277988241,
+      "loss": 6.4542,
+      "mean_token_accuracy": 0.09980635419487953,
+      "num_tokens": 52349150.0,
+      "step": 1800
+    },
+    {
+      "entropy": 6.048123767375946,
+      "epoch": 0.23138738626059221,
+      "grad_norm": 0.31153035163879395,
+      "learning_rate": 0.0019960000277209427,
+      "loss": 6.2618,
+      "mean_token_accuracy": 0.1142095160484314,
+      "num_tokens": 53806391.0,
+      "step": 1850
+    },
+    {
+      "entropy": 5.801566255092621,
+      "epoch": 0.2376410994027704,
+      "grad_norm": 0.5886215567588806,
+      "learning_rate": 0.0019953604530703268,
+      "loss": 6.0319,
+      "mean_token_accuracy": 0.1300203223899007,
+      "num_tokens": 55256012.0,
+      "step": 1900
+    },
+    {
+      "entropy": 5.624916222095489,
+      "epoch": 0.24389481254494857,
+      "grad_norm": 0.39841362833976746,
+      "learning_rate": 0.0019946735844254253,
+      "loss": 5.8955,
+      "mean_token_accuracy": 0.14404148422181606,
+      "num_tokens": 56709946.0,
+      "step": 1950
+    },
+    {
+      "entropy": 5.538290665149689,
+      "epoch": 0.25014852568712675,
+      "grad_norm": 0.45605042576789856,
+      "learning_rate": 0.0019939394544224174,
+      "loss": 5.7458,
+      "mean_token_accuracy": 0.15620031498372555,
+      "num_tokens": 58164585.0,
+      "step": 2000
+    },
+    {
+      "entropy": 5.484496328830719,
+      "epoch": 0.2564022388293049,
+      "grad_norm": 0.3109079897403717,
+      "learning_rate": 0.001993158097943078,
+      "loss": 5.6343,
+      "mean_token_accuracy": 0.16481737934052945,
+      "num_tokens": 59616941.0,
+      "step": 2050
+    },
+    {
+      "entropy": 5.389205462932587,
+      "epoch": 0.26265595197148306,
+      "grad_norm": 0.4438038468360901,
+      "learning_rate": 0.001992329552113123,
+      "loss": 5.5087,
+      "mean_token_accuracy": 0.17319339476525783,
+      "num_tokens": 61069091.0,
+      "step": 2100
+    },
+    {
+      "entropy": 5.315016667842865,
+      "epoch": 0.2689096651136612,
+      "grad_norm": 0.32988205552101135,
+      "learning_rate": 0.001991453856300444,
+      "loss": 5.4307,
+      "mean_token_accuracy": 0.18028867013752461,
+      "num_tokens": 62519727.0,
+      "step": 2150
+    },
+    {
+      "entropy": 5.234848070144653,
+      "epoch": 0.2751633782558394,
+      "grad_norm": 0.4925743341445923,
+      "learning_rate": 0.0019905310521132354,
+      "loss": 5.2956,
+      "mean_token_accuracy": 0.18991463743150233,
+      "num_tokens": 63968440.0,
+      "step": 2200
+    },
+    {
+      "entropy": 5.179865393638611,
+      "epoch": 0.28141709139801757,
+      "grad_norm": 0.5279175639152527,
+      "learning_rate": 0.0019895611833980224,
+      "loss": 5.2495,
+      "mean_token_accuracy": 0.1949171732366085,
+      "num_tokens": 65425112.0,
+      "step": 2250
+    },
+    {
+      "entropy": 5.098879344463349,
+      "epoch": 0.2876708045401957,
+      "grad_norm": 0.2797149121761322,
+      "learning_rate": 0.001988544296237573,
+      "loss": 5.1657,
+      "mean_token_accuracy": 0.20032618649303913,
+      "num_tokens": 66887248.0,
+      "step": 2300
+    },
+    {
+      "entropy": 5.0061674976348876,
+      "epoch": 0.29392451768237393,
+      "grad_norm": 0.31702014803886414,
+      "learning_rate": 0.0019874804389487114,
+      "loss": 5.0679,
+      "mean_token_accuracy": 0.20840579211711885,
+      "num_tokens": 68334156.0,
+      "step": 2350
+    },
+    {
+      "entropy": 4.919540185928344,
+      "epoch": 0.3001782308245521,
+      "grad_norm": 0.3924313485622406,
+      "learning_rate": 0.00198636966208002,
+      "loss": 4.9854,
+      "mean_token_accuracy": 0.214654476121068,
+      "num_tokens": 69791998.0,
+      "step": 2400
+    },
+    {
+      "entropy": 4.897560942173004,
+      "epoch": 0.30643194396673024,
+      "grad_norm": 0.5938014984130859,
+      "learning_rate": 0.0019852120184094403,
+      "loss": 4.9439,
+      "mean_token_accuracy": 0.21875820383429528,
+      "num_tokens": 71248467.0,
+      "step": 2450
+    },
+    {
+      "entropy": 4.851730942726135,
+      "epoch": 0.3126856571089084,
+      "grad_norm": 0.3996203541755676,
+      "learning_rate": 0.001984007562941762,
+      "loss": 4.8655,
+      "mean_token_accuracy": 0.22476962342858314,
+      "num_tokens": 72702130.0,
+      "step": 2500
+    },
+    {
+      "entropy": 4.811781849861145,
+      "epoch": 0.3189393702510866,
+      "grad_norm": 0.3617077171802521,
+      "learning_rate": 0.001982756352906012,
+      "loss": 4.7954,
+      "mean_token_accuracy": 0.23030147060751915,
+      "num_tokens": 74147898.0,
+      "step": 2550
+    },
+    {
+      "entropy": 4.748332953453064,
+      "epoch": 0.32519308339326475,
+      "grad_norm": 0.4124760925769806,
+      "learning_rate": 0.001981458447752735,
+      "loss": 4.7312,
+      "mean_token_accuracy": 0.23577869348227978,
+      "num_tokens": 75596808.0,
+      "step": 2600
+    },
+    {
+      "entropy": 4.711932921409607,
+      "epoch": 0.3314467965354429,
+      "grad_norm": 0.530938982963562,
+      "learning_rate": 0.0019801139091511657,
+      "loss": 4.6903,
+      "mean_token_accuracy": 0.2386609048396349,
+      "num_tokens": 77048990.0,
+      "step": 2650
+    },
+    {
+      "entropy": 4.668835508823395,
+      "epoch": 0.3377005096776211,
+      "grad_norm": 0.37766778469085693,
+      "learning_rate": 0.0019787228009863034,
+      "loss": 4.6045,
+      "mean_token_accuracy": 0.24587701044976712,
+      "num_tokens": 78488093.0,
+      "step": 2700
+    },
+    {
+      "entropy": 4.620059154033661,
+      "epoch": 0.34395422281979926,
+      "grad_norm": 0.5416187644004822,
+      "learning_rate": 0.0019772851893558735,
+      "loss": 4.5806,
+      "mean_token_accuracy": 0.24779064148664476,
+      "num_tokens": 79945023.0,
+      "step": 2750
+    },
+    {
+      "entropy": 4.587407686710358,
+      "epoch": 0.3502079359619774,
+      "grad_norm": 0.37975025177001953,
+      "learning_rate": 0.0019758011425671865,
+      "loss": 4.5253,
+      "mean_token_accuracy": 0.25320823200047016,
+      "num_tokens": 81395329.0,
+      "step": 2800
+    },
+    {
+      "entropy": 4.567973017692566,
+      "epoch": 0.3564616491041556,
+      "grad_norm": 0.4259650707244873,
+      "learning_rate": 0.0019742707311338945,
+      "loss": 4.4934,
+      "mean_token_accuracy": 0.2562766546756029,
+      "num_tokens": 82841834.0,
+      "step": 2850
+    },
+    {
+      "entropy": 4.524812614917755,
+      "epoch": 0.3627153622463338,
+      "grad_norm": 0.4680491089820862,
+      "learning_rate": 0.0019726940277726393,
+      "loss": 4.4345,
+      "mean_token_accuracy": 0.262068510428071,
+      "num_tokens": 84298683.0,
+      "step": 2900
+    },
+    {
+      "entropy": 4.455774722099304,
+      "epoch": 0.3689690753885119,
+      "grad_norm": 0.5032073855400085,
+      "learning_rate": 0.0019710711073995984,
+      "loss": 4.3687,
+      "mean_token_accuracy": 0.2663784135878086,
+      "num_tokens": 85753078.0,
+      "step": 2950
+    },
+    {
+      "entropy": 4.445376222133636,
+      "epoch": 0.3752227885306901,
+      "grad_norm": 0.34912991523742676,
+      "learning_rate": 0.001969402047126923,
+      "loss": 4.3641,
+      "mean_token_accuracy": 0.26783014215528966,
+      "num_tokens": 87205732.0,
+      "step": 3000
+    },
+    {
+      "entropy": 4.4141078424453735,
+      "epoch": 0.3814765016728683,
+      "grad_norm": 0.3609781265258789,
+      "learning_rate": 0.001967686926259077,
+      "loss": 4.3228,
+      "mean_token_accuracy": 0.2722491616755724,
+      "num_tokens": 88665255.0,
+      "step": 3050
+    },
+    {
+      "entropy": 4.394694765806198,
+      "epoch": 0.38773021481504644,
+      "grad_norm": 0.3642273247241974,
+      "learning_rate": 0.0019659258262890683,
+      "loss": 4.2721,
+      "mean_token_accuracy": 0.27585428170859816,
+      "num_tokens": 90121985.0,
+      "step": 3100
+    },
+    {
+      "entropy": 4.3303623342514035,
+      "epoch": 0.3939839279572246,
+      "grad_norm": 0.4377816617488861,
+      "learning_rate": 0.001964118830894575,
+      "loss": 4.216,
+      "mean_token_accuracy": 0.2799024074524641,
+      "num_tokens": 91571242.0,
+      "step": 3150
+    },
+    {
+      "entropy": 4.312202001810074,
+      "epoch": 0.4002376410994028,
+      "grad_norm": 0.4025227129459381,
+      "learning_rate": 0.0019622660259339705,
+      "loss": 4.1866,
+      "mean_token_accuracy": 0.2834921093285084,
+      "num_tokens": 93022711.0,
+      "step": 3200
+    },
+    {
+      "entropy": 4.284852714538574,
+      "epoch": 0.40649135424158095,
+      "grad_norm": 0.34949183464050293,
+      "learning_rate": 0.0019603674994422456,
+      "loss": 4.1723,
+      "mean_token_accuracy": 0.28453252926468847,
+      "num_tokens": 94479555.0,
+      "step": 3250
+    },
+    {
+      "entropy": 4.307129458189011,
+      "epoch": 0.4127450673837591,
+      "grad_norm": 0.40033891797065735,
+      "learning_rate": 0.0019584233416268214,
+      "loss": 4.1644,
+      "mean_token_accuracy": 0.28743029810488224,
+      "num_tokens": 95939600.0,
+      "step": 3300
+    },
+    {
+      "entropy": 4.242270601987839,
+      "epoch": 0.41899878052593725,
+      "grad_norm": 0.36012735962867737,
+      "learning_rate": 0.0019564336448632683,
+      "loss": 4.0949,
+      "mean_token_accuracy": 0.2915411788970232,
+      "num_tokens": 97385909.0,
+      "step": 3350
+    },
+    {
+      "entropy": 4.240862241983414,
+      "epoch": 0.42525249366811546,
+      "grad_norm": 0.41399946808815,
+      "learning_rate": 0.001954398503690913,
+      "loss": 4.0866,
+      "mean_token_accuracy": 0.2938184529542923,
+      "num_tokens": 98835622.0,
+      "step": 3400
+    },
+    {
+      "entropy": 4.196718490123748,
+      "epoch": 0.4315062068102936,
+      "grad_norm": 0.36769694089889526,
+      "learning_rate": 0.0019523180148083485,
+      "loss": 4.0527,
+      "mean_token_accuracy": 0.29610458359122277,
+      "num_tokens": 100288372.0,
+      "step": 3450
+    },
+    {
+      "entropy": 4.174614375829696,
+      "epoch": 0.43775991995247177,
+      "grad_norm": 0.37334468960762024,
+      "learning_rate": 0.001950192277068838,
+      "loss": 4.0218,
+      "mean_token_accuracy": 0.29870147973299027,
+      "num_tokens": 101742863.0,
+      "step": 3500
+    },
+    {
+      "entropy": 4.1543604850769045,
+      "epoch": 0.44401363309465,
+      "grad_norm": 0.4744473993778229,
+      "learning_rate": 0.0019480213914756198,
+      "loss": 3.9855,
+      "mean_token_accuracy": 0.3036028662323952,
+      "num_tokens": 103192353.0,
+      "step": 3550
+    },
+    {
+      "entropy": 4.149127557277679,
+      "epoch": 0.4502673462368281,
+      "grad_norm": 0.46005499362945557,
+      "learning_rate": 0.001945805461177106,
+      "loss": 3.9889,
+      "mean_token_accuracy": 0.3025831463932991,
+      "num_tokens": 104654829.0,
+      "step": 3600
+    },
+    {
+      "entropy": 4.131506505012513,
+      "epoch": 0.4565210593790063,
+      "grad_norm": 0.399476021528244,
+      "learning_rate": 0.0019435445914619835,
+      "loss": 3.9587,
+      "mean_token_accuracy": 0.3061337152123451,
+      "num_tokens": 106112408.0,
+      "step": 3650
+    },
+    {
+      "entropy": 4.12854926109314,
+      "epoch": 0.46277477252118443,
+      "grad_norm": 0.35083433985710144,
+      "learning_rate": 0.0019412388897542105,
+      "loss": 3.9582,
+      "mean_token_accuracy": 0.30483592957258226,
+      "num_tokens": 107568410.0,
+      "step": 3700
+    },
+    {
+      "entropy": 4.099326175451279,
+      "epoch": 0.46902848566336264,
+      "grad_norm": 0.3819263279438019,
+      "learning_rate": 0.0019388884656079118,
+      "loss": 3.9136,
+      "mean_token_accuracy": 0.3103523187339306,
+      "num_tokens": 109023493.0,
+      "step": 3750
+    },
+    {
+      "entropy": 4.095232236385345,
+      "epoch": 0.4752821988055408,
+      "grad_norm": 0.3186725676059723,
+      "learning_rate": 0.0019364934307021732,
+      "loss": 3.8919,
+      "mean_token_accuracy": 0.31130151748657225,
+      "num_tokens": 110472661.0,
+      "step": 3800
+    },
+    {
+      "entropy": 4.082479554414749,
+      "epoch": 0.48153591194771894,
+      "grad_norm": 0.40309175848960876,
+      "learning_rate": 0.0019340538988357374,
+      "loss": 3.8624,
+      "mean_token_accuracy": 0.31485995426774027,
+      "num_tokens": 111928648.0,
+      "step": 3850
+    },
+    {
+      "entropy": 4.07897432565689,
+      "epoch": 0.48778962508989715,
+      "grad_norm": 0.36660367250442505,
+      "learning_rate": 0.001931569985921594,
+      "loss": 3.85,
+      "mean_token_accuracy": 0.31685930326581,
+      "num_tokens": 113391508.0,
+      "step": 3900
+    },
+    {
+      "entropy": 4.041114414930344,
+      "epoch": 0.4940433382320753,
+      "grad_norm": 0.399808794260025,
+      "learning_rate": 0.0019290418099814732,
+      "loss": 3.8116,
+      "mean_token_accuracy": 0.3205082568526268,
+      "num_tokens": 114838971.0,
+      "step": 3950
+    },
+    {
+      "entropy": 4.03851842045784,
+      "epoch": 0.5002970513742535,
+      "grad_norm": 0.36455509066581726,
+      "learning_rate": 0.0019264694911402393,
+      "loss": 3.8159,
+      "mean_token_accuracy": 0.31901825517416,
+      "num_tokens": 116295558.0,
+      "step": 4000
+    },
+    {
+      "entropy": 3.97854936003685,
+      "epoch": 0.5065507645164317,
+      "grad_norm": 0.34177082777023315,
+      "learning_rate": 0.0019238531516201794,
+      "loss": 3.7636,
+      "mean_token_accuracy": 0.3237869530916214,
+      "num_tokens": 117740838.0,
+      "step": 4050
+    },
+    {
+      "entropy": 3.976211417913437,
+      "epoch": 0.5128044776586098,
+      "grad_norm": 0.2913322448730469,
+      "learning_rate": 0.0019211929157352016,
+      "loss": 3.7627,
+      "mean_token_accuracy": 0.32512620434165,
+      "num_tokens": 119187209.0,
+      "step": 4100
+    },
+    {
+      "entropy": 3.98531445145607,
+      "epoch": 0.519058190800788,
+      "grad_norm": 0.3192107081413269,
+      "learning_rate": 0.0019184889098849225,
+      "loss": 3.7607,
+      "mean_token_accuracy": 0.325792491286993,
+      "num_tokens": 120636737.0,
+      "step": 4150
+    },
+    {
+      "entropy": 3.993393454551697,
+      "epoch": 0.5253119039429661,
+      "grad_norm": 0.42490988969802856,
+      "learning_rate": 0.0019157412625486656,
+      "loss": 3.7979,
+      "mean_token_accuracy": 0.3200970686972141,
+      "num_tokens": 122093771.0,
+      "step": 4200
+    },
+    {
+      "entropy": 3.9543004703521727,
+      "epoch": 0.5315656170851443,
+      "grad_norm": 0.3723016679286957,
+      "learning_rate": 0.001912950104279355,
+      "loss": 3.7382,
+      "mean_token_accuracy": 0.3264580935239792,
+      "num_tokens": 123549819.0,
+      "step": 4250
+    },
+    {
+      "entropy": 3.915160721540451,
+      "epoch": 0.5378193302273224,
+      "grad_norm": 0.3779139220714569,
+      "learning_rate": 0.0019101155676973115,
+      "loss": 3.7012,
+      "mean_token_accuracy": 0.33177016422152517,
+      "num_tokens": 124998395.0,
+      "step": 4300
+    },
+    {
+      "entropy": 3.9146971213817596,
+      "epoch": 0.5440730433695007,
+      "grad_norm": 0.3581169843673706,
+      "learning_rate": 0.001907237787483953,
+      "loss": 3.7012,
+      "mean_token_accuracy": 0.33225410625338553,
+      "num_tokens": 126458044.0,
+      "step": 4350
+    },
+    {
+      "entropy": 3.875672823190689,
+      "epoch": 0.5503267565116788,
+      "grad_norm": 0.32717034220695496,
+      "learning_rate": 0.0019043169003753945,
+      "loss": 3.6556,
+      "mean_token_accuracy": 0.336458790153265,
+      "num_tokens": 127906980.0,
+      "step": 4400
+    },
+    {
+      "entropy": 3.879705456495285,
+      "epoch": 0.556580469653857,
+      "grad_norm": 0.32244476675987244,
+      "learning_rate": 0.0019013530451559503,
+      "loss": 3.6516,
+      "mean_token_accuracy": 0.33660307377576826,
+      "num_tokens": 129362058.0,
+      "step": 4450
+    },
+    {
+      "entropy": 3.870090285539627,
+      "epoch": 0.5628341827960351,
+      "grad_norm": 0.3444347083568573,
+      "learning_rate": 0.001898346362651541,
+      "loss": 3.6327,
+      "mean_token_accuracy": 0.3381097550690174,
+      "num_tokens": 130814885.0,
+      "step": 4500
+    },
+    {
+      "entropy": 3.8484600913524627,
+      "epoch": 0.5690878959382133,
+      "grad_norm": 0.3887946605682373,
+      "learning_rate": 0.0018952969957230002,
+      "loss": 3.5848,
+      "mean_token_accuracy": 0.3428233973681927,
+      "num_tokens": 132264698.0,
+      "step": 4550
+    },
+    {
+      "entropy": 3.8668723714351656,
+      "epoch": 0.5753416090803914,
+      "grad_norm": 0.29115742444992065,
+      "learning_rate": 0.0018922050892592904,
+      "loss": 3.6327,
+      "mean_token_accuracy": 0.339210794121027,
+      "num_tokens": 133724718.0,
+      "step": 4600
+    },
+    {
+      "entropy": 3.8426230347156523,
+      "epoch": 0.5815953222225696,
+      "grad_norm": 0.3402599096298218,
+      "learning_rate": 0.0018890707901706146,
+      "loss": 3.5872,
+      "mean_token_accuracy": 0.344546006321907,
+      "num_tokens": 135180844.0,
+      "step": 4650
+    },
+    {
+      "entropy": 3.824489916563034,
+      "epoch": 0.5878490353647479,
+      "grad_norm": 0.31970226764678955,
+      "learning_rate": 0.0018858942473814378,
+      "loss": 3.5729,
+      "mean_token_accuracy": 0.34506093606352806,
+      "num_tokens": 136629379.0,
+      "step": 4700
+    },
+    {
+      "entropy": 3.790500236749649,
+      "epoch": 0.594102748506926,
+      "grad_norm": 0.3910897970199585,
+      "learning_rate": 0.001882675611823411,
+      "loss": 3.5463,
+      "mean_token_accuracy": 0.3468193484842777,
+      "num_tokens": 138088613.0,
+      "step": 4750
+    },
+    {
+      "entropy": 3.840239323377609,
+      "epoch": 0.6003564616491042,
+      "grad_norm": 0.3762884736061096,
+      "learning_rate": 0.0018794150364281998,
+      "loss": 3.5642,
+      "mean_token_accuracy": 0.3468084912002087,
+      "num_tokens": 139542618.0,
+      "step": 4800
+    },
+    {
+      "entropy": 3.8174883902072905,
+      "epoch": 0.6066101747912823,
+      "grad_norm": 0.3109763562679291,
+      "learning_rate": 0.0018761126761202176,
+      "loss": 3.552,
+      "mean_token_accuracy": 0.34760625913739207,
+      "num_tokens": 141007922.0,
+      "step": 4850
+    },
+    {
+      "entropy": 3.7677294433116915,
+      "epoch": 0.6128638879334605,
+      "grad_norm": 0.36054331064224243,
+      "learning_rate": 0.0018727686878092637,
+      "loss": 3.5001,
+      "mean_token_accuracy": 0.35205888241529465,
+      "num_tokens": 142455882.0,
+      "step": 4900
+    },
+    {
+      "entropy": 3.7628906965255737,
+      "epoch": 0.6191176010756386,
+      "grad_norm": 0.36796653270721436,
+      "learning_rate": 0.0018693832303830695,
+      "loss": 3.5106,
+      "mean_token_accuracy": 0.3525958973169327,
+      "num_tokens": 143897412.0,
+      "step": 4950
+    },
+    {
+      "entropy": 3.768322126865387,
+      "epoch": 0.6253713142178168,
+      "grad_norm": 0.3298179507255554,
+      "learning_rate": 0.001865956464699748,
+      "loss": 3.5025,
+      "mean_token_accuracy": 0.351756524592638,
+      "num_tokens": 145343541.0,
+      "step": 5000
+    },
+    {
+      "entropy": 3.7577738535404204,
+      "epoch": 0.631625027359995,
+      "grad_norm": 0.31647345423698425,
+      "learning_rate": 0.001862488553580151,
+      "loss": 3.4857,
+      "mean_token_accuracy": 0.35421416819095614,
+      "num_tokens": 146796685.0,
+      "step": 5050
+    },
+    {
+      "entropy": 3.755051065683365,
+      "epoch": 0.6378787405021732,
+      "grad_norm": 0.3068569600582123,
+      "learning_rate": 0.001858979661800132,
+      "loss": 3.4748,
+      "mean_token_accuracy": 0.3558480441570282,
+      "num_tokens": 148260883.0,
+      "step": 5100
+    },
+    {
+      "entropy": 3.7353078365325927,
+      "epoch": 0.6441324536443513,
+      "grad_norm": 0.3454782962799072,
+      "learning_rate": 0.0018554299560827187,
+      "loss": 3.4677,
+      "mean_token_accuracy": 0.3566476738452911,
+      "num_tokens": 149713216.0,
+      "step": 5150
+    },
+    {
+      "entropy": 3.7258300411701204,
+      "epoch": 0.6503861667865295,
+      "grad_norm": 0.36780253052711487,
+      "learning_rate": 0.0018518396050901892,
+      "loss": 3.4538,
+      "mean_token_accuracy": 0.35792204320430754,
+      "num_tokens": 151165656.0,
+      "step": 5200
+    },
+    {
+      "entropy": 3.73144770860672,
+      "epoch": 0.6566398799287076,
+      "grad_norm": 0.3001205325126648,
+      "learning_rate": 0.0018482087794160593,
+      "loss": 3.4446,
+      "mean_token_accuracy": 0.3578939388692379,
+      "num_tokens": 152612381.0,
+      "step": 5250
+    },
+    {
+      "entropy": 3.6795049965381623,
+      "epoch": 0.6628935930708858,
+      "grad_norm": 0.33437180519104004,
+      "learning_rate": 0.001844537651576977,
+      "loss": 3.4153,
+      "mean_token_accuracy": 0.36169921815395356,
+      "num_tokens": 154065168.0,
+      "step": 5300
+    },
+    {
+      "entropy": 3.716750056743622,
+      "epoch": 0.669147306213064,
+      "grad_norm": 0.32502371072769165,
+      "learning_rate": 0.001840826396004525,
+      "loss": 3.4496,
+      "mean_token_accuracy": 0.3587889949977398,
+      "num_tokens": 155524242.0,
+      "step": 5350
+    },
+    {
+      "entropy": 3.6966857600212095,
+      "epoch": 0.6754010193552422,
+      "grad_norm": 0.30576571822166443,
+      "learning_rate": 0.0018370751890369325,
+      "loss": 3.4046,
+      "mean_token_accuracy": 0.3636187882721424,
+      "num_tokens": 156971839.0,
+      "step": 5400
+    },
+    {
+      "entropy": 3.709896868467331,
+      "epoch": 0.6816547324974204,
+      "grad_norm": 0.3042902946472168,
+      "learning_rate": 0.0018332842089106966,
+      "loss": 3.4222,
+      "mean_token_accuracy": 0.36168786689639093,
+      "num_tokens": 158425753.0,
+      "step": 5450
+    },
+    {
+      "entropy": 3.6710725367069243,
+      "epoch": 0.6879084456395985,
+      "grad_norm": 0.31839218735694885,
+      "learning_rate": 0.001829453635752115,
+      "loss": 3.3889,
+      "mean_token_accuracy": 0.3644601459801197,
+      "num_tokens": 159875972.0,
+      "step": 5500
+    },
+    {
+      "entropy": 3.6800344026088716,
+      "epoch": 0.6941621587817767,
+      "grad_norm": 0.2987898886203766,
+      "learning_rate": 0.0018255836515687243,
+      "loss": 3.3678,
+      "mean_token_accuracy": 0.3673081286251545,
+      "num_tokens": 161331864.0,
+      "step": 5550
+    },
+    {
+      "entropy": 3.6611654114723207,
+      "epoch": 0.7004158719239548,
+      "grad_norm": 0.35385656356811523,
+      "learning_rate": 0.0018216744402406555,
+      "loss": 3.3577,
+      "mean_token_accuracy": 0.36767142444849016,
+      "num_tokens": 162777956.0,
+      "step": 5600
+    },
+    {
+      "entropy": 3.66405290722847,
+      "epoch": 0.706669585066133,
+      "grad_norm": 0.48773252964019775,
+      "learning_rate": 0.0018177261875118947,
+      "loss": 3.3652,
+      "mean_token_accuracy": 0.366995564699173,
+      "num_tokens": 164231330.0,
+      "step": 5650
+    },
+    {
+      "entropy": 3.664638661146164,
+      "epoch": 0.7129232982083112,
+      "grad_norm": 0.32329079508781433,
+      "learning_rate": 0.001813739080981458,
+      "loss": 3.3734,
+      "mean_token_accuracy": 0.3672369138896465,
+      "num_tokens": 165680616.0,
+      "step": 5700
+    },
+    {
+      "entropy": 3.6627173352241518,
+      "epoch": 0.7191770113504894,
+      "grad_norm": 0.3321169316768646,
+      "learning_rate": 0.0018097133100944786,
+      "loss": 3.3584,
+      "mean_token_accuracy": 0.36895839124917984,
+      "num_tokens": 167132788.0,
+      "step": 5750
+    },
+    {
+      "entropy": 3.665049159526825,
+      "epoch": 0.7254307244926675,
+      "grad_norm": 0.37287676334381104,
+      "learning_rate": 0.001805649066133205,
+      "loss": 3.3521,
+      "mean_token_accuracy": 0.36942937210202215,
+      "num_tokens": 168588161.0,
+      "step": 5800
+    },
+    {
+      "entropy": 3.6524035894870757,
+      "epoch": 0.7316844376348457,
+      "grad_norm": 0.2737661302089691,
+      "learning_rate": 0.001801546542207912,
+      "loss": 3.3352,
+      "mean_token_accuracy": 0.3705759882926941,
+      "num_tokens": 170041372.0,
+      "step": 5850
+    },
+    {
+      "entropy": 3.638882428407669,
+      "epoch": 0.7379381507770238,
+      "grad_norm": 0.2713734805583954,
+      "learning_rate": 0.0017974059332477254,
+      "loss": 3.3251,
+      "mean_token_accuracy": 0.37299351826310156,
+      "num_tokens": 171499920.0,
+      "step": 5900
+    },
+    {
+      "entropy": 3.6681057953834535,
+      "epoch": 0.744191863919202,
+      "grad_norm": 0.294450044631958,
+      "learning_rate": 0.0017932274359913599,
+      "loss": 3.3267,
+      "mean_token_accuracy": 0.3729023614525795,
+      "num_tokens": 172950683.0,
+      "step": 5950
+    },
+    {
+      "entropy": 3.644505295753479,
+      "epoch": 0.7504455770613802,
+      "grad_norm": 0.308858722448349,
+      "learning_rate": 0.001789011248977772,
+      "loss": 3.338,
+      "mean_token_accuracy": 0.37171697527170183,
+      "num_tokens": 174406731.0,
+      "step": 6000
+    },
+    {
+      "entropy": 3.616956330537796,
+      "epoch": 0.7566992902035584,
+      "grad_norm": 0.34715735912323,
+      "learning_rate": 0.001784757572536726,
+      "loss": 3.2902,
+      "mean_token_accuracy": 0.3758828853070736,
+      "num_tokens": 175850934.0,
+      "step": 6050
+    },
+    {
+      "entropy": 3.6194920027256012,
+      "epoch": 0.7629530033457366,
+      "grad_norm": 0.3026665449142456,
+      "learning_rate": 0.0017804666087792743,
+      "loss": 3.3085,
+      "mean_token_accuracy": 0.37579055145382884,
+      "num_tokens": 177312757.0,
+      "step": 6100
+    },
+    {
+      "entropy": 3.6085926043987273,
+      "epoch": 0.7692067164879147,
+      "grad_norm": 0.3407142758369446,
+      "learning_rate": 0.001776138561588156,
+      "loss": 3.2887,
+      "mean_token_accuracy": 0.37644373178482055,
+      "num_tokens": 178768968.0,
+      "step": 6150
+    },
+    {
+      "entropy": 3.5980601501464844,
+      "epoch": 0.7754604296300929,
+      "grad_norm": 0.30068376660346985,
+      "learning_rate": 0.0017717736366081086,
+      "loss": 3.2662,
+      "mean_token_accuracy": 0.3782395477592945,
+      "num_tokens": 180224489.0,
+      "step": 6200
+    },
+    {
+      "entropy": 3.564172661304474,
+      "epoch": 0.781714142772271,
+      "grad_norm": 0.31278517842292786,
+      "learning_rate": 0.0017673720412360971,
+      "loss": 3.2472,
+      "mean_token_accuracy": 0.38103450983762743,
+      "num_tokens": 181675705.0,
+      "step": 6250
+    },
+    {
+      "entropy": 3.572136676311493,
+      "epoch": 0.7879678559144492,
+      "grad_norm": 0.36869823932647705,
+      "learning_rate": 0.0017629339846114598,
+      "loss": 3.2544,
+      "mean_token_accuracy": 0.3806936974823475,
+      "num_tokens": 183131543.0,
+      "step": 6300
+    },
+    {
+      "entropy": 3.5726712560653686,
+      "epoch": 0.7942215690566273,
+      "grad_norm": 0.28678107261657715,
+      "learning_rate": 0.0017584596776059707,
+      "loss": 3.266,
+      "mean_token_accuracy": 0.37899156510829923,
+      "num_tokens": 184585889.0,
+      "step": 6350
+    },
+    {
+      "entropy": 3.5974609780311586,
+      "epoch": 0.8004752821988056,
+      "grad_norm": 0.2944851219654083,
+      "learning_rate": 0.0017539493328138206,
+      "loss": 3.2432,
+      "mean_token_accuracy": 0.38106390431523324,
+      "num_tokens": 186029456.0,
+      "step": 6400
+    },
+    {
+      "entropy": 3.5803931736946106,
+      "epoch": 0.8067289953409837,
+      "grad_norm": 0.9102575182914734,
+      "learning_rate": 0.0017494031645415157,
+      "loss": 3.2312,
+      "mean_token_accuracy": 0.38339566633105276,
+      "num_tokens": 187477525.0,
+      "step": 6450
+    },
+    {
+      "entropy": 3.6113105177879334,
+      "epoch": 0.8129827084831619,
+      "grad_norm": 0.29048532247543335,
+      "learning_rate": 0.0017448213887976936,
+      "loss": 3.3103,
+      "mean_token_accuracy": 0.3724804125726223,
+      "num_tokens": 188930874.0,
+      "step": 6500
+    },
+    {
+      "entropy": 3.545231056213379,
+      "epoch": 0.81923642162534,
+      "grad_norm": 0.2970048189163208,
+      "learning_rate": 0.0017402042232828629,
+      "loss": 3.2126,
+      "mean_token_accuracy": 0.38453035309910777,
+      "num_tokens": 190377981.0,
+      "step": 6550
+    },
+    {
+      "entropy": 3.536184823513031,
+      "epoch": 0.8254901347675182,
+      "grad_norm": 0.30343198776245117,
+      "learning_rate": 0.0017355518873790555,
+      "loss": 3.223,
+      "mean_token_accuracy": 0.3839683771133423,
+      "num_tokens": 191831429.0,
+      "step": 6600
+    },
+    {
+      "entropy": 3.5653406488895416,
+      "epoch": 0.8317438479096964,
+      "grad_norm": 0.300936758518219,
+      "learning_rate": 0.001730864602139406,
+      "loss": 3.2313,
+      "mean_token_accuracy": 0.38390363290905954,
+      "num_tokens": 193283746.0,
+      "step": 6650
+    },
+    {
+      "entropy": 3.5464817702770235,
+      "epoch": 0.8379975610518745,
+      "grad_norm": 0.25732260942459106,
+      "learning_rate": 0.001726142590277646,
+      "loss": 3.1963,
+      "mean_token_accuracy": 0.3856934300065041,
+      "num_tokens": 194733765.0,
+      "step": 6700
+    },
+    {
+      "entropy": 3.5280859065055847,
+      "epoch": 0.8442512741940528,
+      "grad_norm": 0.28379517793655396,
+      "learning_rate": 0.0017213860761575242,
+      "loss": 3.1849,
+      "mean_token_accuracy": 0.38866120934486387,
+      "num_tokens": 196181881.0,
+      "step": 6750
+    },
+    {
+      "entropy": 3.5327327394485475,
+      "epoch": 0.8505049873362309,
+      "grad_norm": 0.2904554307460785,
+      "learning_rate": 0.001716595285782144,
+      "loss": 3.1818,
+      "mean_token_accuracy": 0.38906206741929056,
+      "num_tokens": 197633452.0,
+      "step": 6800
+    },
+    {
+      "entropy": 3.5628380298614504,
+      "epoch": 0.8567587004784091,
+      "grad_norm": 0.26626577973365784,
+      "learning_rate": 0.0017117704467832263,
+      "loss": 3.2098,
+      "mean_token_accuracy": 0.38593448519706725,
+      "num_tokens": 199091645.0,
+      "step": 6850
+    },
+    {
+      "entropy": 3.524178329706192,
+      "epoch": 0.8630124136205872,
+      "grad_norm": 0.28848186135292053,
+      "learning_rate": 0.0017069117884102925,
+      "loss": 3.1794,
+      "mean_token_accuracy": 0.3886136619746685,
+      "num_tokens": 200550605.0,
+      "step": 6900
+    },
+    {
+      "entropy": 3.512812074422836,
+      "epoch": 0.8692661267627654,
+      "grad_norm": 0.2841011881828308,
+      "learning_rate": 0.0017020195415197736,
+      "loss": 3.161,
+      "mean_token_accuracy": 0.39136223718523977,
+      "num_tokens": 201989189.0,
+      "step": 6950
+    },
+    {
+      "entropy": 3.518073334693909,
+      "epoch": 0.8755198399049435,
+      "grad_norm": 0.2641650140285492,
+      "learning_rate": 0.0016970939385640396,
+      "loss": 3.1677,
+      "mean_token_accuracy": 0.3900047905743122,
+      "num_tokens": 203444023.0,
+      "step": 7000
+    },
+    {
+      "entropy": 3.512884209156036,
+      "epoch": 0.8817735530471217,
+      "grad_norm": 0.3092753291130066,
+      "learning_rate": 0.0016921352135803559,
+      "loss": 3.1787,
+      "mean_token_accuracy": 0.3905393898487091,
+      "num_tokens": 204898253.0,
+      "step": 7050
+    },
+    {
+      "entropy": 3.4959859812259673,
+      "epoch": 0.8880272661893,
+      "grad_norm": 0.2931444048881531,
+      "learning_rate": 0.0016871436021797615,
+      "loss": 3.1478,
+      "mean_token_accuracy": 0.3930986747145653,
+      "num_tokens": 206355062.0,
+      "step": 7100
+    },
+    {
+      "entropy": 3.494699535369873,
+      "epoch": 0.8942809793314781,
+      "grad_norm": 0.2965364158153534,
+      "learning_rate": 0.0016821193415358753,
+      "loss": 3.1468,
+      "mean_token_accuracy": 0.39268109157681463,
+      "num_tokens": 207812565.0,
+      "step": 7150
+    },
+    {
+      "entropy": 3.5158278048038483,
+      "epoch": 0.9005346924736563,
+      "grad_norm": 0.2810881435871124,
+      "learning_rate": 0.0016770626703736275,
+      "loss": 3.1539,
+      "mean_token_accuracy": 0.39324876174330714,
+      "num_tokens": 209264738.0,
+      "step": 7200
+    },
+    {
+      "entropy": 3.516902258396149,
+      "epoch": 0.9067884056158344,
+      "grad_norm": 0.25858649611473083,
+      "learning_rate": 0.0016719738289579155,
+      "loss": 3.1521,
+      "mean_token_accuracy": 0.39297880709171296,
+      "num_tokens": 210725805.0,
+      "step": 7250
+    },
+    {
+      "entropy": 3.521990531682968,
+      "epoch": 0.9130421187580126,
+      "grad_norm": 0.2878429889678955,
+      "learning_rate": 0.0016668530590821882,
+      "loss": 3.1219,
+      "mean_token_accuracy": 0.3965709181129932,
+      "num_tokens": 212175877.0,
+      "step": 7300
+    },
+    {
+      "entropy": 3.5328981733322142,
+      "epoch": 0.9192958319001907,
+      "grad_norm": 0.25898534059524536,
+      "learning_rate": 0.0016617006040569571,
+      "loss": 3.1404,
+      "mean_token_accuracy": 0.3935855607688427,
+      "num_tokens": 213623949.0,
+      "step": 7350
+    },
+    {
+      "entropy": 3.5099762165546418,
+      "epoch": 0.9255495450423689,
+      "grad_norm": 0.2775290012359619,
+      "learning_rate": 0.001656516708698236,
+      "loss": 3.1184,
+      "mean_token_accuracy": 0.3956405273079872,
+      "num_tokens": 215068031.0,
+      "step": 7400
+    },
+    {
+      "entropy": 3.524550086259842,
+      "epoch": 0.9318032581845471,
+      "grad_norm": 0.26542389392852783,
+      "learning_rate": 0.0016513016193159089,
+      "loss": 3.1305,
+      "mean_token_accuracy": 0.3949857813119888,
+      "num_tokens": 216520762.0,
+      "step": 7450
+    },
+    {
+      "entropy": 3.52587819814682,
+      "epoch": 0.9380569713267253,
+      "grad_norm": 0.2635110020637512,
+      "learning_rate": 0.001646055583702026,
+      "loss": 3.1235,
+      "mean_token_accuracy": 0.3954600378870964,
+      "num_tokens": 217972779.0,
+      "step": 7500
+    },
+    {
+      "entropy": 3.5055534410476685,
+      "epoch": 0.9443106844689034,
+      "grad_norm": 0.2407570630311966,
+      "learning_rate": 0.0016407788511190308,
+      "loss": 3.0997,
+      "mean_token_accuracy": 0.3990290623903274,
+      "num_tokens": 219418956.0,
+      "step": 7550
+    },
+    {
+      "entropy": 3.491668597459793,
+      "epoch": 0.9505643976110816,
+      "grad_norm": 0.2876957058906555,
+      "learning_rate": 0.0016354716722879151,
+      "loss": 3.1016,
+      "mean_token_accuracy": 0.3992904122173786,
+      "num_tokens": 220868170.0,
+      "step": 7600
+    },
+    {
+      "entropy": 3.4823805701732637,
+      "epoch": 0.9568181107532597,
+      "grad_norm": 0.26311132311820984,
+      "learning_rate": 0.001630134299376308,
+      "loss": 3.1059,
+      "mean_token_accuracy": 0.39762111693620683,
+      "num_tokens": 222320108.0,
+      "step": 7650
+    },
+    {
+      "entropy": 3.4473106372356415,
+      "epoch": 0.9630718238954379,
+      "grad_norm": 0.27912265062332153,
+      "learning_rate": 0.0016247669859864933,
+      "loss": 3.0641,
+      "mean_token_accuracy": 0.40164967700839044,
+      "num_tokens": 223771680.0,
+      "step": 7700
+    },
+    {
+      "entropy": 3.462431426048279,
+      "epoch": 0.969325537037616,
+      "grad_norm": 0.26055315136909485,
+      "learning_rate": 0.0016193699871433595,
+      "loss": 3.0906,
+      "mean_token_accuracy": 0.40069493383169175,
+      "num_tokens": 225224312.0,
+      "step": 7750
+    },
+    {
+      "entropy": 3.46071582198143,
+      "epoch": 0.9755792501797943,
+      "grad_norm": 0.2583518624305725,
+      "learning_rate": 0.0016139435592822837,
+      "loss": 3.0814,
+      "mean_token_accuracy": 0.40042845577001573,
+      "num_tokens": 226669853.0,
+      "step": 7800
+    },
+    {
+      "entropy": 3.429822801351547,
+      "epoch": 0.9818329633219725,
+      "grad_norm": 0.24933992326259613,
+      "learning_rate": 0.0016084879602369449,
+      "loss": 3.0642,
+      "mean_token_accuracy": 0.40294322282075884,
+      "num_tokens": 228118458.0,
+      "step": 7850
+    },
+    {
+      "entropy": 3.462758048772812,
+      "epoch": 0.9880866764641506,
+      "grad_norm": 0.2879364490509033,
+      "learning_rate": 0.0016030034492270752,
+      "loss": 3.0928,
+      "mean_token_accuracy": 0.40064197540283203,
+      "num_tokens": 229576123.0,
+      "step": 7900
+    },
+    {
+      "entropy": 3.457154052257538,
+      "epoch": 0.9943403896063288,
+      "grad_norm": 0.2881210148334503,
+      "learning_rate": 0.0015974902868461432,
+      "loss": 3.0798,
+      "mean_token_accuracy": 0.4007870285212994,
+      "num_tokens": 231024565.0,
+      "step": 7950
+    },
+    {
+      "entropy": 3.424220819763726,
+      "epoch": 1.0005002970513743,
+      "grad_norm": 0.2757372260093689,
+      "learning_rate": 0.0015919487350489704,
+      "loss": 3.0722,
+      "mean_token_accuracy": 0.402178642713479,
+      "num_tokens": 232448848.0,
+      "step": 8000
+    },
+    {
+      "entropy": 3.4124892342090605,
+      "epoch": 1.0067540101935524,
+      "grad_norm": 0.2726467251777649,
+      "learning_rate": 0.0015863790571392854,
+      "loss": 3.0441,
+      "mean_token_accuracy": 0.4046463569998741,
+      "num_tokens": 233902055.0,
+      "step": 8050
+    },
+    {
+      "entropy": 3.424123123884201,
+      "epoch": 1.0130077233357306,
+      "grad_norm": 0.2699504494667053,
+      "learning_rate": 0.0015807815177572148,
+      "loss": 3.0545,
+      "mean_token_accuracy": 0.40355317264795304,
+      "num_tokens": 235361552.0,
+      "step": 8100
+    },
+    {
+      "entropy": 3.410174704790115,
+      "epoch": 1.0192614364779087,
+      "grad_norm": 0.26823437213897705,
+      "learning_rate": 0.0015751563828667052,
+      "loss": 3.0393,
+      "mean_token_accuracy": 0.40559999957680704,
+      "num_tokens": 236812470.0,
+      "step": 8150
+    },
+    {
+      "entropy": 3.4348238849639894,
+      "epoch": 1.025515149620087,
+      "grad_norm": 0.24912285804748535,
+      "learning_rate": 0.0015695039197428902,
+      "loss": 3.0611,
+      "mean_token_accuracy": 0.4035358209908009,
+      "num_tokens": 238271687.0,
+      "step": 8200
+    },
+    {
+      "entropy": 3.420966486930847,
+      "epoch": 1.0317688627622652,
+      "grad_norm": 0.27924877405166626,
+      "learning_rate": 0.0015638243969593884,
+      "loss": 3.0306,
+      "mean_token_accuracy": 0.4060466636717319,
+      "num_tokens": 239728605.0,
+      "step": 8250
+    },
+    {
+      "entropy": 3.414804896116257,
+      "epoch": 1.0380225759044432,
+      "grad_norm": 0.279109388589859,
+      "learning_rate": 0.0015581180843755424,
+      "loss": 3.0462,
+      "mean_token_accuracy": 0.4052661190927029,
+      "num_tokens": 241186564.0,
+      "step": 8300
+    },
+    {
+      "entropy": 3.3893780303001404,
+      "epoch": 1.0442762890466215,
+      "grad_norm": 0.25905993580818176,
+      "learning_rate": 0.0015523852531235983,
+      "loss": 2.9909,
+      "mean_token_accuracy": 0.4110270082950592,
+      "num_tokens": 242636402.0,
+      "step": 8350
+    },
+    {
+      "entropy": 3.3776789891719816,
+      "epoch": 1.0505300021887995,
+      "grad_norm": 0.26641029119491577,
+      "learning_rate": 0.0015466261755958208,
+      "loss": 3.0057,
+      "mean_token_accuracy": 0.4096148502826691,
+      "num_tokens": 244087527.0,
+      "step": 8400
+    },
+    {
+      "entropy": 3.407488120794296,
+      "epoch": 1.0567837153309778,
+      "grad_norm": 0.2628111243247986,
+      "learning_rate": 0.0015408411254315526,
+      "loss": 3.0423,
+      "mean_token_accuracy": 0.4053249025344849,
+      "num_tokens": 245542169.0,
+      "step": 8450
+    },
+    {
+      "entropy": 3.4050830852985383,
+      "epoch": 1.0630374284731559,
+      "grad_norm": 0.23083467781543732,
+      "learning_rate": 0.001535030377504211,
+      "loss": 3.0253,
+      "mean_token_accuracy": 0.40700640618801115,
+      "num_tokens": 246988548.0,
+      "step": 8500
+    },
+    {
+      "entropy": 3.3786729192733764,
+      "epoch": 1.0692911416153341,
+      "grad_norm": 0.2786962389945984,
+      "learning_rate": 0.0015291942079082284,
+      "loss": 3.0021,
+      "mean_token_accuracy": 0.4086934958398342,
+      "num_tokens": 248438834.0,
+      "step": 8550
+    },
+    {
+      "entropy": 3.389970999956131,
+      "epoch": 1.0755448547575122,
+      "grad_norm": 0.24150459468364716,
+      "learning_rate": 0.0015233328939459332,
+      "loss": 3.0186,
+      "mean_token_accuracy": 0.4084386694431305,
+      "num_tokens": 249893987.0,
+      "step": 8600
+    },
+    {
+      "entropy": 3.379815950393677,
+      "epoch": 1.0817985678996904,
+      "grad_norm": 0.23742859065532684,
+      "learning_rate": 0.0015174467141143748,
+      "loss": 2.9962,
+      "mean_token_accuracy": 0.4109345510601997,
+      "num_tokens": 251346496.0,
+      "step": 8650
+    },
+    {
+      "entropy": 3.381633528470993,
+      "epoch": 1.0880522810418687,
+      "grad_norm": 0.2817932963371277,
+      "learning_rate": 0.001511535948092091,
+      "loss": 3.0132,
+      "mean_token_accuracy": 0.40931278571486474,
+      "num_tokens": 252812600.0,
+      "step": 8700
+    },
+    {
+      "entropy": 3.37952179312706,
+      "epoch": 1.0943059941840467,
+      "grad_norm": 0.23492373526096344,
+      "learning_rate": 0.0015056008767258162,
+      "loss": 2.9865,
+      "mean_token_accuracy": 0.4107097062468529,
+      "num_tokens": 254265989.0,
+      "step": 8750
+    },
+    {
+      "entropy": 3.3976906645298004,
+      "epoch": 1.100559707326225,
+      "grad_norm": 0.26069802045822144,
+      "learning_rate": 0.0014996417820171426,
+      "loss": 2.9959,
+      "mean_token_accuracy": 0.4109515391290188,
+      "num_tokens": 255715028.0,
+      "step": 8800
+    },
+    {
+      "entropy": 3.368416168689728,
+      "epoch": 1.106813420468403,
+      "grad_norm": 0.2870071232318878,
+      "learning_rate": 0.0014936589471091165,
+      "loss": 2.9703,
+      "mean_token_accuracy": 0.41341917037963866,
+      "num_tokens": 257163834.0,
+      "step": 8850
+    },
+    {
+      "entropy": 3.364442890882492,
+      "epoch": 1.1130671336105813,
+      "grad_norm": 0.2520281970500946,
+      "learning_rate": 0.0014876526562727867,
+      "loss": 2.9763,
+      "mean_token_accuracy": 0.41378984034061433,
+      "num_tokens": 258616558.0,
+      "step": 8900
+    },
+    {
+      "entropy": 3.399234428405762,
+      "epoch": 1.1193208467527596,
+      "grad_norm": 0.4310092628002167,
+      "learning_rate": 0.0014816231948936967,
+      "loss": 3.0143,
+      "mean_token_accuracy": 0.4086450488865376,
+      "num_tokens": 260073476.0,
+      "step": 8950
+    },
+    {
+      "entropy": 3.3841825568675996,
+      "epoch": 1.1255745598949376,
+      "grad_norm": 0.2929483652114868,
+      "learning_rate": 0.0014755708494583269,
+      "loss": 3.0013,
+      "mean_token_accuracy": 0.4088966536521912,
+      "num_tokens": 261517515.0,
+      "step": 9000
+    },
+    {
+      "entropy": 3.364690970182419,
+      "epoch": 1.1318282730371159,
+      "grad_norm": 0.2556741535663605,
+      "learning_rate": 0.0014694959075404793,
+      "loss": 2.9746,
+      "mean_token_accuracy": 0.4140892958641052,
+      "num_tokens": 262963815.0,
+      "step": 9050
+    },
+    {
+      "entropy": 3.335093698501587,
+      "epoch": 1.138081986179294,
+      "grad_norm": 0.26782119274139404,
+      "learning_rate": 0.001463398657787616,
+      "loss": 2.9489,
+      "mean_token_accuracy": 0.4161834093928337,
+      "num_tokens": 264419565.0,
+      "step": 9100
+    },
+    {
+      "entropy": 3.340893853902817,
+      "epoch": 1.1443356993214722,
+      "grad_norm": 0.25481972098350525,
+      "learning_rate": 0.001457279389907143,
+      "loss": 2.9554,
+      "mean_token_accuracy": 0.4155861519277096,
+      "num_tokens": 265866694.0,
+      "step": 9150
+    },
+    {
+      "entropy": 3.381483224630356,
+      "epoch": 1.1505894124636502,
+      "grad_norm": 0.2730478346347809,
+      "learning_rate": 0.001451138394652645,
+      "loss": 2.9674,
+      "mean_token_accuracy": 0.41388784766197206,
+      "num_tokens": 267323470.0,
+      "step": 9200
+    },
+    {
+      "entropy": 3.3568492233753204,
+      "epoch": 1.1568431256058285,
+      "grad_norm": 0.2593781650066376,
+      "learning_rate": 0.001444975963810072,
+      "loss": 2.9531,
+      "mean_token_accuracy": 0.4166987505555153,
+      "num_tokens": 268771908.0,
+      "step": 9250
+    },
+    {
+      "entropy": 3.343725650310516,
+      "epoch": 1.1630968387480065,
+      "grad_norm": 0.23936976492404938,
+      "learning_rate": 0.001438792390183872,
+      "loss": 2.9497,
+      "mean_token_accuracy": 0.4166214270889759,
+      "num_tokens": 270222216.0,
+      "step": 9300
+    },
+    {
+      "entropy": 3.3452346408367157,
+      "epoch": 1.1693505518901848,
+      "grad_norm": 0.24291479587554932,
+      "learning_rate": 0.0014325879675830824,
+      "loss": 2.9781,
+      "mean_token_accuracy": 0.4135470025241375,
+      "num_tokens": 271676400.0,
+      "step": 9350
+    },
+    {
+      "entropy": 3.3736480152606965,
+      "epoch": 1.175604265032363,
+      "grad_norm": 0.2399204820394516,
+      "learning_rate": 0.0014263629908073663,
+      "loss": 2.9887,
+      "mean_token_accuracy": 0.4122758340835571,
+      "num_tokens": 273129248.0,
+      "step": 9400
+    },
+    {
+      "entropy": 3.335775601863861,
+      "epoch": 1.181857978174541,
+      "grad_norm": 0.24937410652637482,
+      "learning_rate": 0.001420117755633008,
+      "loss": 2.9449,
+      "mean_token_accuracy": 0.4171235427260399,
+      "num_tokens": 274577598.0,
+      "step": 9450
+    },
+    {
+      "entropy": 3.326780825853348,
+      "epoch": 1.1881116913167193,
+      "grad_norm": 0.24946729838848114,
+      "learning_rate": 0.0014138525587988585,
+      "loss": 2.9499,
+      "mean_token_accuracy": 0.41634685471653937,
+      "num_tokens": 276035839.0,
+      "step": 9500
+    },
+    {
+      "entropy": 3.347506259679794,
+      "epoch": 1.1943654044588974,
+      "grad_norm": 0.232426717877388,
+      "learning_rate": 0.0014075676979922347,
+      "loss": 2.9388,
+      "mean_token_accuracy": 0.41728449061512946,
+      "num_tokens": 277483534.0,
+      "step": 9550
+    },
+    {
+      "entropy": 3.3232869446277618,
+      "epoch": 1.2006191176010756,
+      "grad_norm": 0.2653588354587555,
+      "learning_rate": 0.0014012634718347774,
+      "loss": 2.9378,
+      "mean_token_accuracy": 0.41797628834843636,
+      "num_tokens": 278942090.0,
+      "step": 9600
+    },
+    {
+      "entropy": 3.338738178014755,
+      "epoch": 1.206872830743254,
+      "grad_norm": 0.23284447193145752,
+      "learning_rate": 0.00139494017986826,
+      "loss": 2.9277,
+      "mean_token_accuracy": 0.41921328961849214,
+      "num_tokens": 280386565.0,
+      "step": 9650
+    },
+    {
+      "entropy": 3.3239081263542176,
+      "epoch": 1.213126543885432,
+      "grad_norm": 0.231287881731987,
+      "learning_rate": 0.0013885981225403574,
+      "loss": 2.9221,
+      "mean_token_accuracy": 0.42032514050602915,
+      "num_tokens": 281837049.0,
+      "step": 9700
+    },
+    {
+      "entropy": 3.3193623530864715,
+      "epoch": 1.2193802570276102,
+      "grad_norm": 0.2553178071975708,
+      "learning_rate": 0.0013822376011903717,
+      "loss": 2.9251,
+      "mean_token_accuracy": 0.41874496802687644,
+      "num_tokens": 283291625.0,
+      "step": 9750
+    },
+    {
+      "entropy": 3.313581793308258,
+      "epoch": 1.2256339701697883,
+      "grad_norm": 0.2445349544286728,
+      "learning_rate": 0.0013758589180349107,
+      "loss": 2.9036,
+      "mean_token_accuracy": 0.4213790266215801,
+      "num_tokens": 284744533.0,
+      "step": 9800
+    },
+    {
+      "entropy": 3.333759317398071,
+      "epoch": 1.2318876833119665,
+      "grad_norm": 0.23976311087608337,
+      "learning_rate": 0.0013694623761535305,
+      "loss": 2.9223,
+      "mean_token_accuracy": 0.41979889243841173,
+      "num_tokens": 286202783.0,
+      "step": 9850
+    },
+    {
+      "entropy": 3.3275194442272187,
+      "epoch": 1.2381413964541446,
+      "grad_norm": 0.24221397936344147,
+      "learning_rate": 0.0013630482794743356,
+      "loss": 2.9227,
+      "mean_token_accuracy": 0.42032407999038696,
+      "num_tokens": 287655421.0,
+      "step": 9900
+    },
+    {
+      "entropy": 3.297724051475525,
+      "epoch": 1.2443951095963228,
+      "grad_norm": 0.24389943480491638,
+      "learning_rate": 0.001356616932759536,
+      "loss": 2.9053,
+      "mean_token_accuracy": 0.422286706417799,
+      "num_tokens": 289116534.0,
+      "step": 9950
+    },
+    {
+      "entropy": 3.316089181900024,
+      "epoch": 1.2506488227385009,
+      "grad_norm": 0.2706010937690735,
+      "learning_rate": 0.0013501686415909678,
+      "loss": 2.91,
+      "mean_token_accuracy": 0.42257274031639097,
+      "num_tokens": 290575760.0,
+      "step": 10000
+    },
+    {
+      "entropy": 3.3069709300994874,
+      "epoch": 1.2569025358806791,
+      "grad_norm": 0.2567020058631897,
+      "learning_rate": 0.0013437037123555737,
+      "loss": 2.9068,
+      "mean_token_accuracy": 0.42272754848003385,
+      "num_tokens": 292030604.0,
+      "step": 10050
+    },
+    {
+      "entropy": 3.322487668991089,
+      "epoch": 1.2631562490228574,
+      "grad_norm": 0.24985256791114807,
+      "learning_rate": 0.0013372224522308438,
+      "loss": 2.9249,
+      "mean_token_accuracy": 0.41974695593118666,
+      "num_tokens": 293486083.0,
+      "step": 10100
+    },
+    {
+      "entropy": 3.3206483924388888,
+      "epoch": 1.2694099621650354,
+      "grad_norm": 0.2233990728855133,
+      "learning_rate": 0.001330725169170222,
+      "loss": 2.9117,
+      "mean_token_accuracy": 0.4212852735817432,
+      "num_tokens": 294937867.0,
+      "step": 10150
+    },
+    {
+      "entropy": 3.3343569612503052,
+      "epoch": 1.2756636753072137,
+      "grad_norm": 0.2400444597005844,
+      "learning_rate": 0.0013242121718884728,
+      "loss": 2.9097,
+      "mean_token_accuracy": 0.42152070671319963,
+      "num_tokens": 296390576.0,
+      "step": 10200
+    },
+    {
+      "entropy": 3.3441244554519653,
+      "epoch": 1.2819173884493917,
+      "grad_norm": 0.24485696852207184,
+      "learning_rate": 0.0013176837698470139,
+      "loss": 2.8971,
+      "mean_token_accuracy": 0.4235718835890293,
+      "num_tokens": 297842021.0,
+      "step": 10250
+    },
+    {
+      "entropy": 3.318438186645508,
+      "epoch": 1.28817110159157,
+      "grad_norm": 0.23577703535556793,
+      "learning_rate": 0.0013111402732392096,
+      "loss": 2.9088,
+      "mean_token_accuracy": 0.4217331087589264,
+      "num_tokens": 299298294.0,
+      "step": 10300
+    },
+    {
+      "entropy": 3.3001376008987426,
+      "epoch": 1.2944248147337483,
+      "grad_norm": 0.2448028177022934,
+      "learning_rate": 0.0013045819929756364,
+      "loss": 2.8789,
+      "mean_token_accuracy": 0.4240961664915085,
+      "num_tokens": 300755464.0,
+      "step": 10350
+    },
+    {
+      "entropy": 3.3169038486480713,
+      "epoch": 1.3006785278759263,
+      "grad_norm": 0.24094216525554657,
+      "learning_rate": 0.0012980092406693062,
+      "loss": 2.8981,
+      "mean_token_accuracy": 0.4229963217675686,
+      "num_tokens": 302205274.0,
+      "step": 10400
+    },
+    {
+      "entropy": 3.306311628818512,
+      "epoch": 1.3069322410181046,
+      "grad_norm": 0.23586605489253998,
+      "learning_rate": 0.0012914223286208625,
+      "loss": 2.8739,
+      "mean_token_accuracy": 0.4253669346868992,
+      "num_tokens": 303659297.0,
+      "step": 10450
+    },
+    {
+      "entropy": 3.3153705585002897,
+      "epoch": 1.3131859541602826,
+      "grad_norm": 0.22188012301921844,
+      "learning_rate": 0.0012848215698037415,
+      "loss": 2.8784,
+      "mean_token_accuracy": 0.42567239463329315,
+      "num_tokens": 305117516.0,
+      "step": 10500
+    },
+    {
+      "entropy": 3.349429177045822,
+      "epoch": 1.3194396673024609,
+      "grad_norm": 0.24746544659137726,
+      "learning_rate": 0.0012782072778493002,
+      "loss": 2.9087,
+      "mean_token_accuracy": 0.42274825111031533,
+      "num_tokens": 306570303.0,
+      "step": 10550
+    },
+    {
+      "entropy": 3.329492164850235,
+      "epoch": 1.3256933804446391,
+      "grad_norm": 0.24711163341999054,
+      "learning_rate": 0.0012715797670319147,
+      "loss": 2.8892,
+      "mean_token_accuracy": 0.42441680446267127,
+      "num_tokens": 308030555.0,
+      "step": 10600
+    },
+    {
+      "entropy": 3.3335850477218627,
+      "epoch": 1.3319470935868172,
+      "grad_norm": 0.2298772633075714,
+      "learning_rate": 0.001264939352254049,
+      "loss": 2.8933,
+      "mean_token_accuracy": 0.42399927601218224,
+      "num_tokens": 309482666.0,
+      "step": 10650
+    },
+    {
+      "entropy": 3.3008330821990968,
+      "epoch": 1.3382008067289952,
+      "grad_norm": 0.23629966378211975,
+      "learning_rate": 0.0012582863490312908,
+      "loss": 2.8679,
+      "mean_token_accuracy": 0.4268447032570839,
+      "num_tokens": 310933867.0,
+      "step": 10700
+    },
+    {
+      "entropy": 3.294624637365341,
+      "epoch": 1.3444545198711735,
+      "grad_norm": 0.23950351774692535,
+      "learning_rate": 0.001251621073477361,
+      "loss": 2.8606,
+      "mean_token_accuracy": 0.4277565456926823,
+      "num_tokens": 312390391.0,
+      "step": 10750
+    },
+    {
+      "entropy": 3.290150513648987,
+      "epoch": 1.3507082330133517,
+      "grad_norm": 0.24080321192741394,
+      "learning_rate": 0.0012449438422890936,
+      "loss": 2.8666,
+      "mean_token_accuracy": 0.4271367233991623,
+      "num_tokens": 313838259.0,
+      "step": 10800
+    },
+    {
+      "entropy": 3.2781748247146605,
+      "epoch": 1.3569619461555298,
+      "grad_norm": 0.24540862441062927,
+      "learning_rate": 0.0012382549727313875,
+      "loss": 2.8558,
+      "mean_token_accuracy": 0.4279210121929646,
+      "num_tokens": 315286075.0,
+      "step": 10850
+    },
+    {
+      "entropy": 3.2989360880851746,
+      "epoch": 1.363215659297708,
+      "grad_norm": 0.2313542664051056,
+      "learning_rate": 0.0012315547826221318,
+      "loss": 2.8597,
+      "mean_token_accuracy": 0.4279498742520809,
+      "num_tokens": 316736678.0,
+      "step": 10900
+    },
+    {
+      "entropy": 3.304781684875488,
+      "epoch": 1.369469372439886,
+      "grad_norm": 0.2552511990070343,
+      "learning_rate": 0.001224843590317107,
+      "loss": 2.8853,
+      "mean_token_accuracy": 0.42454988047480585,
+      "num_tokens": 318188717.0,
+      "step": 10950
+    },
+    {
+      "entropy": 3.2968351674079894,
+      "epoch": 1.3757230855820644,
+      "grad_norm": 0.23053553700447083,
+      "learning_rate": 0.001218121714694855,
+      "loss": 2.8576,
+      "mean_token_accuracy": 0.42793465360999106,
+      "num_tokens": 319650569.0,
+      "step": 11000
+    },
+    {
+      "entropy": 3.2916909861564636,
+      "epoch": 1.3819767987242426,
+      "grad_norm": 0.24563711881637573,
+      "learning_rate": 0.0012113894751415296,
+      "loss": 2.8673,
+      "mean_token_accuracy": 0.42703422278165815,
+      "num_tokens": 321109384.0,
+      "step": 11050
+    },
+    {
+      "entropy": 3.2736746430397035,
+      "epoch": 1.3882305118664207,
+      "grad_norm": 0.24409718811511993,
+      "learning_rate": 0.001204647191535722,
+      "loss": 2.8325,
+      "mean_token_accuracy": 0.43039547055959704,
+      "num_tokens": 322562775.0,
+      "step": 11100
+    },
+    {
+      "entropy": 3.295790330171585,
+      "epoch": 1.394484225008599,
+      "grad_norm": 0.22487126290798187,
+      "learning_rate": 0.0011978951842332606,
+      "loss": 2.8457,
+      "mean_token_accuracy": 0.4293624579906464,
+      "num_tokens": 324014136.0,
+      "step": 11150
+    },
+    {
+      "entropy": 3.2935370481014252,
+      "epoch": 1.400737938150777,
+      "grad_norm": 0.25449955463409424,
+      "learning_rate": 0.0011911337740519892,
+      "loss": 2.8508,
+      "mean_token_accuracy": 0.4287599550187588,
+      "num_tokens": 325469987.0,
+      "step": 11200
+    },
+    {
+      "entropy": 3.2637720847129823,
+      "epoch": 1.4069916512929552,
+      "grad_norm": 0.23394949734210968,
+      "learning_rate": 0.001184363282256525,
+      "loss": 2.8295,
+      "mean_token_accuracy": 0.43097158893942833,
+      "num_tokens": 326922931.0,
+      "step": 11250
+    },
+    {
+      "entropy": 3.2709325647354124,
+      "epoch": 1.4132453644351335,
+      "grad_norm": 0.229343444108963,
+      "learning_rate": 0.0011775840305429921,
+      "loss": 2.8306,
+      "mean_token_accuracy": 0.4322802956402302,
+      "num_tokens": 328381052.0,
+      "step": 11300
+    },
+    {
+      "entropy": 3.2761396980285644,
+      "epoch": 1.4194990775773115,
+      "grad_norm": 0.23448783159255981,
+      "learning_rate": 0.0011707963410237379,
+      "loss": 2.8363,
+      "mean_token_accuracy": 0.430686649531126,
+      "num_tokens": 329830903.0,
+      "step": 11350
+    },
+    {
+      "entropy": 3.2788108825683593,
+      "epoch": 1.4257527907194896,
+      "grad_norm": 0.2271973043680191,
+      "learning_rate": 0.001164000536212027,
+      "loss": 2.8335,
+      "mean_token_accuracy": 0.43118971526622774,
+      "num_tokens": 331280393.0,
+      "step": 11400
+    },
+    {
+      "entropy": 3.2778335607051847,
+      "epoch": 1.4320065038616678,
+      "grad_norm": 0.2223415970802307,
+      "learning_rate": 0.001157196939006718,
+      "loss": 2.8424,
+      "mean_token_accuracy": 0.43043203085660936,
+      "num_tokens": 332733626.0,
+      "step": 11450
+    },
+    {
+      "entropy": 3.275426481962204,
+      "epoch": 1.438260217003846,
+      "grad_norm": 0.2389313280582428,
+      "learning_rate": 0.0011503858726769202,
+      "loss": 2.836,
+      "mean_token_accuracy": 0.43069657027721403,
+      "num_tokens": 334190416.0,
+      "step": 11500
+    },
+    {
+      "entropy": 3.2625485575199127,
+      "epoch": 1.4445139301460241,
+      "grad_norm": 0.22828622162342072,
+      "learning_rate": 0.0011435676608466346,
+      "loss": 2.8426,
+      "mean_token_accuracy": 0.4305486410856247,
+      "num_tokens": 335637372.0,
+      "step": 11550
+    },
+    {
+      "entropy": 3.2681053030490874,
+      "epoch": 1.4507676432882024,
+      "grad_norm": 0.25355732440948486,
+      "learning_rate": 0.0011367426274793762,
+      "loss": 2.8428,
+      "mean_token_accuracy": 0.4308210875093937,
+      "num_tokens": 337083595.0,
+      "step": 11600
+    },
+    {
+      "entropy": 3.2613851368427276,
+      "epoch": 1.4570213564303804,
+      "grad_norm": 0.2454511970281601,
+      "learning_rate": 0.001129911096862782,
+      "loss": 2.8138,
+      "mean_token_accuracy": 0.4339980843663216,
+      "num_tokens": 338542359.0,
+      "step": 11650
+    },
+    {
+      "entropy": 3.2610177218914034,
+      "epoch": 1.4632750695725587,
+      "grad_norm": 0.21777866780757904,
+      "learning_rate": 0.0011230733935932023,
+      "loss": 2.817,
+      "mean_token_accuracy": 0.43292332217097285,
+      "num_tokens": 339988394.0,
+      "step": 11700
+    },
+    {
+      "entropy": 3.25565652012825,
+      "epoch": 1.469528782714737,
+      "grad_norm": 0.2365877777338028,
+      "learning_rate": 0.0011162298425602764,
+      "loss": 2.8198,
+      "mean_token_accuracy": 0.43387064009904863,
+      "num_tokens": 341446604.0,
+      "step": 11750
+    },
+    {
+      "entropy": 3.249692702293396,
+      "epoch": 1.475782495856915,
+      "grad_norm": 0.22539696097373962,
+      "learning_rate": 0.001109380768931498,
+      "loss": 2.8068,
+      "mean_token_accuracy": 0.4340075485408306,
+      "num_tokens": 342900120.0,
+      "step": 11800
+    },
+    {
+      "entropy": 3.2544299829006196,
+      "epoch": 1.4820362089990933,
+      "grad_norm": 0.23777642846107483,
+      "learning_rate": 0.0011025264981367638,
+      "loss": 2.8062,
+      "mean_token_accuracy": 0.43426609471440314,
+      "num_tokens": 344353638.0,
+      "step": 11850
+    },
+    {
+      "entropy": 3.2589793014526367,
+      "epoch": 1.4882899221412713,
+      "grad_norm": 0.23899991810321808,
+      "learning_rate": 0.00109566735585291,
+      "loss": 2.8147,
+      "mean_token_accuracy": 0.4332495957612991,
+      "num_tokens": 345800869.0,
+      "step": 11900
+    },
+    {
+      "entropy": 3.2570550775527956,
+      "epoch": 1.4945436352834496,
+      "grad_norm": 0.22672854363918304,
+      "learning_rate": 0.0010888036679882395,
+      "loss": 2.8023,
+      "mean_token_accuracy": 0.4355665297806263,
+      "num_tokens": 347250646.0,
+      "step": 11950
+    },
+    {
+      "entropy": 3.254550014734268,
+      "epoch": 1.5007973484256278,
+      "grad_norm": 0.2114313691854477,
+      "learning_rate": 0.001081935760667036,
+      "loss": 2.8136,
+      "mean_token_accuracy": 0.43408931702375414,
+      "num_tokens": 348704005.0,
+      "step": 12000
+    },
+    {
+      "entropy": 3.243316088914871,
+      "epoch": 1.5070510615678059,
+      "grad_norm": 0.2516964077949524,
+      "learning_rate": 0.0010750639602140686,
+      "loss": 2.8009,
+      "mean_token_accuracy": 0.43518534049391744,
+      "num_tokens": 350158394.0,
+      "step": 12050
+    },
+    {
+      "entropy": 3.2478549814224245,
+      "epoch": 1.513304774709984,
+      "grad_norm": 0.23279394209384918,
+      "learning_rate": 0.0010681885931390852,
+      "loss": 2.7857,
+      "mean_token_accuracy": 0.4367074476182461,
+      "num_tokens": 351604127.0,
+      "step": 12100
+    },
+    {
+      "entropy": 3.2347246026992797,
+      "epoch": 1.5195584878521622,
+      "grad_norm": 0.21781018376350403,
+      "learning_rate": 0.0010613099861213012,
+      "loss": 2.7956,
+      "mean_token_accuracy": 0.4359207484126091,
+      "num_tokens": 353057864.0,
+      "step": 12150
+    },
+    {
+      "entropy": 3.2461864519119263,
+      "epoch": 1.5258122009943405,
+      "grad_norm": 0.21265676617622375,
+      "learning_rate": 0.0010544284659938746,
+      "loss": 2.7892,
+      "mean_token_accuracy": 0.43556260600686075,
+      "num_tokens": 354506147.0,
+      "step": 12200
+    },
+    {
+      "entropy": 3.2419194054603575,
+      "epoch": 1.5320659141365187,
+      "grad_norm": 0.22998523712158203,
+      "learning_rate": 0.0010475443597283795,
+      "loss": 2.7783,
+      "mean_token_accuracy": 0.43743231013417244,
+      "num_tokens": 355963386.0,
+      "step": 12250
+    },
+    {
+      "entropy": 3.246154465675354,
+      "epoch": 1.5383196272786968,
+      "grad_norm": 0.22558872401714325,
+      "learning_rate": 0.0010406579944192685,
+      "loss": 2.7831,
+      "mean_token_accuracy": 0.43715057760477066,
+      "num_tokens": 357409816.0,
+      "step": 12300
+    },
+    {
+      "entropy": 3.256463017463684,
+      "epoch": 1.5445733404208748,
+      "grad_norm": 0.20943056046962738,
+      "learning_rate": 0.0010337696972683307,
+      "loss": 2.7837,
+      "mean_token_accuracy": 0.4375229972600937,
+      "num_tokens": 358857684.0,
+      "step": 12350
+    },
+    {
+      "entropy": 3.2448647558689117,
+      "epoch": 1.550827053563053,
+      "grad_norm": 0.23157542943954468,
+      "learning_rate": 0.0010268797955691462,
+      "loss": 2.7912,
+      "mean_token_accuracy": 0.43746368065476415,
+      "num_tokens": 360317443.0,
+      "step": 12400
+    },
+    {
+      "entropy": 3.227845176458359,
+      "epoch": 1.5570807667052313,
+      "grad_norm": 0.21983113884925842,
+      "learning_rate": 0.0010199886166915347,
+      "loss": 2.7695,
+      "mean_token_accuracy": 0.43851295456290246,
+      "num_tokens": 361763696.0,
+      "step": 12450
+    },
+    {
+      "entropy": 3.207278673648834,
+      "epoch": 1.5633344798474094,
+      "grad_norm": 0.22169895470142365,
+      "learning_rate": 0.0010130964880659997,
+      "loss": 2.758,
+      "mean_token_accuracy": 0.4404232783615589,
+      "num_tokens": 363215595.0,
+      "step": 12500
+    },
+    {
+      "entropy": 3.2540294277668,
+      "epoch": 1.5695881929895874,
+      "grad_norm": 0.23538173735141754,
+      "learning_rate": 0.001006203737168171,
+      "loss": 2.8065,
+      "mean_token_accuracy": 0.43551131546497346,
+      "num_tokens": 364675498.0,
+      "step": 12550
+    },
+    {
+      "entropy": 3.231663987636566,
+      "epoch": 1.5758419061317657,
+      "grad_norm": 0.2294698804616928,
+      "learning_rate": 0.0009993106915032468,
+      "loss": 2.7788,
+      "mean_token_accuracy": 0.4382545882463455,
+      "num_tokens": 366125218.0,
+      "step": 12600
+    },
+    {
+      "entropy": 3.2363407254219054,
+      "epoch": 1.582095619273944,
+      "grad_norm": 0.2398029863834381,
+      "learning_rate": 0.0009924176785904293,
+      "loss": 2.7851,
+      "mean_token_accuracy": 0.4379922252893448,
+      "num_tokens": 367579050.0,
+      "step": 12650
+    },
+    {
+      "entropy": 3.2210785961151123,
+      "epoch": 1.5883493324161222,
+      "grad_norm": 0.23287877440452576,
+      "learning_rate": 0.0009855250259473657,
+      "loss": 2.7663,
+      "mean_token_accuracy": 0.4396293942630291,
+      "num_tokens": 369033121.0,
+      "step": 12700
+    },
+    {
+      "entropy": 3.2457301235198974,
+      "epoch": 1.5946030455583002,
+      "grad_norm": 0.2148071825504303,
+      "learning_rate": 0.0009786330610745849,
+      "loss": 2.7688,
+      "mean_token_accuracy": 0.43925670653581617,
+      "num_tokens": 370481491.0,
+      "step": 12750
+    },
+    {
+      "entropy": 3.2246074092388155,
+      "epoch": 1.6008567587004783,
+      "grad_norm": 0.21147137880325317,
+      "learning_rate": 0.0009717421114399361,
+      "loss": 2.7438,
+      "mean_token_accuracy": 0.44269678577780724,
+      "num_tokens": 371931424.0,
+      "step": 12800
+    },
+    {
+      "entropy": 3.2194683921337126,
+      "epoch": 1.6071104718426565,
+      "grad_norm": 0.23948106169700623,
+      "learning_rate": 0.0009648525044630312,
+      "loss": 2.7567,
+      "mean_token_accuracy": 0.4407572849094868,
+      "num_tokens": 373377913.0,
+      "step": 12850
+    },
+    {
+      "entropy": 3.241425403356552,
+      "epoch": 1.6133641849848348,
+      "grad_norm": 0.21832682192325592,
+      "learning_rate": 0.0009579645674996858,
+      "loss": 2.781,
+      "mean_token_accuracy": 0.43758947104215623,
+      "num_tokens": 374840821.0,
+      "step": 12900
+    },
+    {
+      "entropy": 3.232071034908295,
+      "epoch": 1.619617898127013,
+      "grad_norm": 0.2332869917154312,
+      "learning_rate": 0.0009510786278263655,
+      "loss": 2.7789,
+      "mean_token_accuracy": 0.43771066397428515,
+      "num_tokens": 376297301.0,
+      "step": 12950
+    },
+    {
+      "entropy": 3.1825023066997526,
+      "epoch": 1.625871611269191,
+      "grad_norm": 0.24876029789447784,
+      "learning_rate": 0.0009441950126246364,
+      "loss": 2.7299,
+      "mean_token_accuracy": 0.4447443017363548,
+      "num_tokens": 377735071.0,
+      "step": 13000
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 23988,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.7708908779142513e+18,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1cb03720ebf32d7010ac6d465f20cddd3cda8eadfa07984c314d1da5144925bc
+size 6225