Model save
Browse files- README.md +68 -0
- all_results.json +8 -0
- generation_config.json +6 -0
- train_results.json +8 -0
- trainer_state.json +2787 -0
README.md
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
base_model: Qwen/Qwen2.5-3B
|
| 3 |
+
library_name: transformers
|
| 4 |
+
model_name: Qwen2.5-3B-Open-R1-GRPO-math-selected-cosine-noRW
|
| 5 |
+
tags:
|
| 6 |
+
- generated_from_trainer
|
| 7 |
+
- trl
|
| 8 |
+
- grpo
|
| 9 |
+
licence: license
|
| 10 |
+
---
|
| 11 |
+
|
| 12 |
+
# Model Card for Qwen2.5-3B-Open-R1-GRPO-math-selected-cosine-noRW
|
| 13 |
+
|
| 14 |
+
This model is a fine-tuned version of [Qwen/Qwen2.5-3B](https://huggingface.co/Qwen/Qwen2.5-3B).
|
| 15 |
+
It has been trained using [TRL](https://github.com/huggingface/trl).
|
| 16 |
+
|
| 17 |
+
## Quick start
|
| 18 |
+
|
| 19 |
+
```python
|
| 20 |
+
from transformers import pipeline
|
| 21 |
+
|
| 22 |
+
question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
|
| 23 |
+
generator = pipeline("text-generation", model="Lansechen/Qwen2.5-3B-Open-R1-GRPO-math-selected-cosine-noRW", device="cuda")
|
| 24 |
+
output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
|
| 25 |
+
print(output["generated_text"])
|
| 26 |
+
```
|
| 27 |
+
|
| 28 |
+
## Training procedure
|
| 29 |
+
|
| 30 |
+
[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/chenran1995-the-chinese-university-of-hong-kong/huggingface/runs/jximl91v)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).
|
| 34 |
+
|
| 35 |
+
### Framework versions
|
| 36 |
+
|
| 37 |
+
- TRL: 0.16.0
|
| 38 |
+
- Transformers: 4.50.0
|
| 39 |
+
- Pytorch: 2.5.1+cu121
|
| 40 |
+
- Datasets: 3.5.0
|
| 41 |
+
- Tokenizers: 0.21.1
|
| 42 |
+
|
| 43 |
+
## Citations
|
| 44 |
+
|
| 45 |
+
Cite GRPO as:
|
| 46 |
+
|
| 47 |
+
```bibtex
|
| 48 |
+
@article{zhihong2024deepseekmath,
|
| 49 |
+
title = {{DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models}},
|
| 50 |
+
author = {Zhihong Shao and Peiyi Wang and Qihao Zhu and Runxin Xu and Junxiao Song and Mingchuan Zhang and Y. K. Li and Y. Wu and Daya Guo},
|
| 51 |
+
year = 2024,
|
| 52 |
+
eprint = {arXiv:2402.03300},
|
| 53 |
+
}
|
| 54 |
+
|
| 55 |
+
```
|
| 56 |
+
|
| 57 |
+
Cite TRL as:
|
| 58 |
+
|
| 59 |
+
```bibtex
|
| 60 |
+
@misc{vonwerra2022trl,
|
| 61 |
+
title = {{TRL: Transformer Reinforcement Learning}},
|
| 62 |
+
author = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallouédec},
|
| 63 |
+
year = 2020,
|
| 64 |
+
journal = {GitHub repository},
|
| 65 |
+
publisher = {GitHub},
|
| 66 |
+
howpublished = {\url{https://github.com/huggingface/trl}}
|
| 67 |
+
}
|
| 68 |
+
```
|
all_results.json
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"total_flos": 0.0,
|
| 3 |
+
"train_loss": -0.019965038798853137,
|
| 4 |
+
"train_runtime": 31425.8764,
|
| 5 |
+
"train_samples": 11040,
|
| 6 |
+
"train_samples_per_second": 0.703,
|
| 7 |
+
"train_steps_per_second": 0.006
|
| 8 |
+
}
|
generation_config.json
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bos_token_id": 151643,
|
| 3 |
+
"eos_token_id": 151643,
|
| 4 |
+
"max_new_tokens": 2048,
|
| 5 |
+
"transformers_version": "4.50.0"
|
| 6 |
+
}
|
train_results.json
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"total_flos": 0.0,
|
| 3 |
+
"train_loss": -0.019965038798853137,
|
| 4 |
+
"train_runtime": 31425.8764,
|
| 5 |
+
"train_samples": 11040,
|
| 6 |
+
"train_samples_per_second": 0.703,
|
| 7 |
+
"train_steps_per_second": 0.006
|
| 8 |
+
}
|
trainer_state.json
ADDED
|
@@ -0,0 +1,2787 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"best_global_step": null,
|
| 3 |
+
"best_metric": null,
|
| 4 |
+
"best_model_checkpoint": null,
|
| 5 |
+
"epoch": 1.9936628643852978,
|
| 6 |
+
"eval_steps": 100,
|
| 7 |
+
"global_step": 196,
|
| 8 |
+
"is_hyper_param_search": false,
|
| 9 |
+
"is_local_process_zero": true,
|
| 10 |
+
"is_world_process_zero": true,
|
| 11 |
+
"log_history": [
|
| 12 |
+
{
|
| 13 |
+
"clip_ratio": 0.0,
|
| 14 |
+
"completion_length": 544.7745819091797,
|
| 15 |
+
"epoch": 0.010139416983523447,
|
| 16 |
+
"grad_norm": 0.2261611670255661,
|
| 17 |
+
"learning_rate": 5e-08,
|
| 18 |
+
"loss": -0.2592,
|
| 19 |
+
"num_tokens": 636118.0,
|
| 20 |
+
"reward": -0.6724561750888824,
|
| 21 |
+
"reward_std": 0.4402390718460083,
|
| 22 |
+
"rewards/cosine_scaled_reward": -0.6758043766021729,
|
| 23 |
+
"rewards/format_reward": 0.003348214435391128,
|
| 24 |
+
"step": 1
|
| 25 |
+
},
|
| 26 |
+
{
|
| 27 |
+
"clip_ratio": 0.0,
|
| 28 |
+
"completion_length": 549.2812843322754,
|
| 29 |
+
"epoch": 0.020278833967046894,
|
| 30 |
+
"grad_norm": 0.22706830501556396,
|
| 31 |
+
"learning_rate": 1e-07,
|
| 32 |
+
"loss": -0.2276,
|
| 33 |
+
"num_tokens": 1263330.0,
|
| 34 |
+
"reward": -0.6239746958017349,
|
| 35 |
+
"reward_std": 0.49842649325728416,
|
| 36 |
+
"rewards/cosine_scaled_reward": -0.6317871585488319,
|
| 37 |
+
"rewards/format_reward": 0.007812500349245965,
|
| 38 |
+
"step": 2
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"clip_ratio": 0.0,
|
| 42 |
+
"completion_length": 569.3448944091797,
|
| 43 |
+
"epoch": 0.030418250950570342,
|
| 44 |
+
"grad_norm": 0.18996624648571014,
|
| 45 |
+
"learning_rate": 1.5e-07,
|
| 46 |
+
"loss": -0.2165,
|
| 47 |
+
"num_tokens": 1913583.0,
|
| 48 |
+
"reward": -0.614225223660469,
|
| 49 |
+
"reward_std": 0.4692864343523979,
|
| 50 |
+
"rewards/cosine_scaled_reward": -0.6231537759304047,
|
| 51 |
+
"rewards/format_reward": 0.008928571827709675,
|
| 52 |
+
"step": 3
|
| 53 |
+
},
|
| 54 |
+
{
|
| 55 |
+
"clip_ratio": 0.0,
|
| 56 |
+
"completion_length": 539.3951110839844,
|
| 57 |
+
"epoch": 0.04055766793409379,
|
| 58 |
+
"grad_norm": 0.22352048754692078,
|
| 59 |
+
"learning_rate": 2e-07,
|
| 60 |
+
"loss": -0.2585,
|
| 61 |
+
"num_tokens": 2533833.0,
|
| 62 |
+
"reward": -0.6429695598781109,
|
| 63 |
+
"reward_std": 0.4445586446672678,
|
| 64 |
+
"rewards/cosine_scaled_reward": -0.6507820263504982,
|
| 65 |
+
"rewards/format_reward": 0.007812500349245965,
|
| 66 |
+
"step": 4
|
| 67 |
+
},
|
| 68 |
+
{
|
| 69 |
+
"clip_ratio": 0.0,
|
| 70 |
+
"completion_length": 576.1462287902832,
|
| 71 |
+
"epoch": 0.050697084917617236,
|
| 72 |
+
"grad_norm": 0.21681766211986542,
|
| 73 |
+
"learning_rate": 2.5e-07,
|
| 74 |
+
"loss": -0.2409,
|
| 75 |
+
"num_tokens": 3176844.0,
|
| 76 |
+
"reward": -0.6119992136955261,
|
| 77 |
+
"reward_std": 0.4665570706129074,
|
| 78 |
+
"rewards/cosine_scaled_reward": -0.6164634898304939,
|
| 79 |
+
"rewards/format_reward": 0.004464285913854837,
|
| 80 |
+
"step": 5
|
| 81 |
+
},
|
| 82 |
+
{
|
| 83 |
+
"clip_ratio": 0.0,
|
| 84 |
+
"completion_length": 570.2154197692871,
|
| 85 |
+
"epoch": 0.060836501901140684,
|
| 86 |
+
"grad_norm": 0.21843321621418,
|
| 87 |
+
"learning_rate": 3e-07,
|
| 88 |
+
"loss": -0.2408,
|
| 89 |
+
"num_tokens": 3816949.0,
|
| 90 |
+
"reward": -0.5715240500867367,
|
| 91 |
+
"reward_std": 0.5433307699859142,
|
| 92 |
+
"rewards/cosine_scaled_reward": -0.5771043859422207,
|
| 93 |
+
"rewards/format_reward": 0.005580357392318547,
|
| 94 |
+
"step": 6
|
| 95 |
+
},
|
| 96 |
+
{
|
| 97 |
+
"clip_ratio": 0.0,
|
| 98 |
+
"completion_length": 539.3672180175781,
|
| 99 |
+
"epoch": 0.07097591888466413,
|
| 100 |
+
"grad_norm": 0.21752215921878815,
|
| 101 |
+
"learning_rate": 3.5e-07,
|
| 102 |
+
"loss": -0.1852,
|
| 103 |
+
"num_tokens": 4424646.0,
|
| 104 |
+
"reward": -0.5407312288880348,
|
| 105 |
+
"reward_std": 0.5957661867141724,
|
| 106 |
+
"rewards/cosine_scaled_reward": -0.5518919117748737,
|
| 107 |
+
"rewards/format_reward": 0.011160714668221772,
|
| 108 |
+
"step": 7
|
| 109 |
+
},
|
| 110 |
+
{
|
| 111 |
+
"clip_ratio": 0.0,
|
| 112 |
+
"completion_length": 558.9989128112793,
|
| 113 |
+
"epoch": 0.08111533586818757,
|
| 114 |
+
"grad_norm": 0.21230322122573853,
|
| 115 |
+
"learning_rate": 4e-07,
|
| 116 |
+
"loss": -0.205,
|
| 117 |
+
"num_tokens": 5053061.0,
|
| 118 |
+
"reward": -0.6614980250597,
|
| 119 |
+
"reward_std": 0.47641005739569664,
|
| 120 |
+
"rewards/cosine_scaled_reward": -0.667078360915184,
|
| 121 |
+
"rewards/format_reward": 0.005580357392318547,
|
| 122 |
+
"step": 8
|
| 123 |
+
},
|
| 124 |
+
{
|
| 125 |
+
"clip_ratio": 0.0,
|
| 126 |
+
"completion_length": 576.405158996582,
|
| 127 |
+
"epoch": 0.09125475285171103,
|
| 128 |
+
"grad_norm": 0.22808963060379028,
|
| 129 |
+
"learning_rate": 4.5e-07,
|
| 130 |
+
"loss": -0.2588,
|
| 131 |
+
"num_tokens": 5692848.0,
|
| 132 |
+
"reward": -0.6227121576666832,
|
| 133 |
+
"reward_std": 0.45661213248968124,
|
| 134 |
+
"rewards/cosine_scaled_reward": -0.6294085532426834,
|
| 135 |
+
"rewards/format_reward": 0.006696428870782256,
|
| 136 |
+
"step": 9
|
| 137 |
+
},
|
| 138 |
+
{
|
| 139 |
+
"clip_ratio": 0.0,
|
| 140 |
+
"completion_length": 516.9732475280762,
|
| 141 |
+
"epoch": 0.10139416983523447,
|
| 142 |
+
"grad_norm": 0.2245020717382431,
|
| 143 |
+
"learning_rate": 5e-07,
|
| 144 |
+
"loss": -0.1928,
|
| 145 |
+
"num_tokens": 6282784.0,
|
| 146 |
+
"reward": -0.5244979597628117,
|
| 147 |
+
"reward_std": 0.5999544225633144,
|
| 148 |
+
"rewards/cosine_scaled_reward": -0.5278461389243603,
|
| 149 |
+
"rewards/format_reward": 0.003348214435391128,
|
| 150 |
+
"step": 10
|
| 151 |
+
},
|
| 152 |
+
{
|
| 153 |
+
"clip_ratio": 0.0,
|
| 154 |
+
"completion_length": 583.3047180175781,
|
| 155 |
+
"epoch": 0.11153358681875793,
|
| 156 |
+
"grad_norm": 0.22737731039524078,
|
| 157 |
+
"learning_rate": 5.5e-07,
|
| 158 |
+
"loss": -0.2092,
|
| 159 |
+
"num_tokens": 6942961.0,
|
| 160 |
+
"reward": -0.609170563519001,
|
| 161 |
+
"reward_std": 0.47449312917888165,
|
| 162 |
+
"rewards/cosine_scaled_reward": -0.6125187538564205,
|
| 163 |
+
"rewards/format_reward": 0.003348214435391128,
|
| 164 |
+
"step": 11
|
| 165 |
+
},
|
| 166 |
+
{
|
| 167 |
+
"clip_ratio": 0.0,
|
| 168 |
+
"completion_length": 587.389533996582,
|
| 169 |
+
"epoch": 0.12167300380228137,
|
| 170 |
+
"grad_norm": 0.22877997159957886,
|
| 171 |
+
"learning_rate": 6e-07,
|
| 172 |
+
"loss": -0.2051,
|
| 173 |
+
"num_tokens": 7600574.0,
|
| 174 |
+
"reward": -0.5761966481804848,
|
| 175 |
+
"reward_std": 0.47745756432414055,
|
| 176 |
+
"rewards/cosine_scaled_reward": -0.5884733907878399,
|
| 177 |
+
"rewards/format_reward": 0.012276786146685481,
|
| 178 |
+
"step": 12
|
| 179 |
+
},
|
| 180 |
+
{
|
| 181 |
+
"clip_ratio": 0.0,
|
| 182 |
+
"completion_length": 605.9844055175781,
|
| 183 |
+
"epoch": 0.13181242078580482,
|
| 184 |
+
"grad_norm": 0.20151640474796295,
|
| 185 |
+
"learning_rate": 6.5e-07,
|
| 186 |
+
"loss": -0.1612,
|
| 187 |
+
"num_tokens": 8271720.0,
|
| 188 |
+
"reward": -0.5711147747933865,
|
| 189 |
+
"reward_std": 0.5248831808567047,
|
| 190 |
+
"rewards/cosine_scaled_reward": -0.5789272524416447,
|
| 191 |
+
"rewards/format_reward": 0.007812500349245965,
|
| 192 |
+
"step": 13
|
| 193 |
+
},
|
| 194 |
+
{
|
| 195 |
+
"clip_ratio": 0.0,
|
| 196 |
+
"completion_length": 554.7109680175781,
|
| 197 |
+
"epoch": 0.14195183776932827,
|
| 198 |
+
"grad_norm": 0.28496649861335754,
|
| 199 |
+
"learning_rate": 7e-07,
|
| 200 |
+
"loss": -0.1814,
|
| 201 |
+
"num_tokens": 8900917.0,
|
| 202 |
+
"reward": -0.5377125255763531,
|
| 203 |
+
"reward_std": 0.589600820094347,
|
| 204 |
+
"rewards/cosine_scaled_reward": -0.5481268018484116,
|
| 205 |
+
"rewards/format_reward": 0.004464285913854837,
|
| 206 |
+
"step": 14
|
| 207 |
+
},
|
| 208 |
+
{
|
| 209 |
+
"clip_ratio": 0.0,
|
| 210 |
+
"completion_length": 597.1194534301758,
|
| 211 |
+
"epoch": 0.1520912547528517,
|
| 212 |
+
"grad_norm": 0.20674261450767517,
|
| 213 |
+
"learning_rate": 7.5e-07,
|
| 214 |
+
"loss": -0.1812,
|
| 215 |
+
"num_tokens": 9564376.0,
|
| 216 |
+
"reward": -0.5252576433122158,
|
| 217 |
+
"reward_std": 0.5734145380556583,
|
| 218 |
+
"rewards/cosine_scaled_reward": -0.5308379791676998,
|
| 219 |
+
"rewards/format_reward": 0.005580357392318547,
|
| 220 |
+
"step": 15
|
| 221 |
+
},
|
| 222 |
+
{
|
| 223 |
+
"clip_ratio": 0.0,
|
| 224 |
+
"completion_length": 587.7846221923828,
|
| 225 |
+
"epoch": 0.16223067173637515,
|
| 226 |
+
"grad_norm": 0.18769149482250214,
|
| 227 |
+
"learning_rate": 8e-07,
|
| 228 |
+
"loss": -0.1947,
|
| 229 |
+
"num_tokens": 10217959.0,
|
| 230 |
+
"reward": -0.5038141719996929,
|
| 231 |
+
"reward_std": 0.6175409816205502,
|
| 232 |
+
"rewards/cosine_scaled_reward": -0.5071623660624027,
|
| 233 |
+
"rewards/format_reward": 0.003348214435391128,
|
| 234 |
+
"step": 16
|
| 235 |
+
},
|
| 236 |
+
{
|
| 237 |
+
"clip_ratio": 0.0,
|
| 238 |
+
"completion_length": 625.3683319091797,
|
| 239 |
+
"epoch": 0.17237008871989862,
|
| 240 |
+
"grad_norm": 0.3610371947288513,
|
| 241 |
+
"learning_rate": 8.499999999999999e-07,
|
| 242 |
+
"loss": -0.1733,
|
| 243 |
+
"num_tokens": 10912801.0,
|
| 244 |
+
"reward": -0.5226632095873356,
|
| 245 |
+
"reward_std": 0.5681972987949848,
|
| 246 |
+
"rewards/cosine_scaled_reward": -0.5282435454428196,
|
| 247 |
+
"rewards/format_reward": 0.005580357392318547,
|
| 248 |
+
"step": 17
|
| 249 |
+
},
|
| 250 |
+
{
|
| 251 |
+
"clip_ratio": 0.0,
|
| 252 |
+
"completion_length": 632.5993499755859,
|
| 253 |
+
"epoch": 0.18250950570342206,
|
| 254 |
+
"grad_norm": 0.183369979262352,
|
| 255 |
+
"learning_rate": 9e-07,
|
| 256 |
+
"loss": -0.1304,
|
| 257 |
+
"num_tokens": 11602954.0,
|
| 258 |
+
"reward": -0.4399515353143215,
|
| 259 |
+
"reward_std": 0.6363755837082863,
|
| 260 |
+
"rewards/cosine_scaled_reward": -0.44553186371922493,
|
| 261 |
+
"rewards/format_reward": 0.005580357392318547,
|
| 262 |
+
"step": 18
|
| 263 |
+
},
|
| 264 |
+
{
|
| 265 |
+
"clip_ratio": 0.0,
|
| 266 |
+
"completion_length": 607.8593978881836,
|
| 267 |
+
"epoch": 0.1926489226869455,
|
| 268 |
+
"grad_norm": 0.18773187696933746,
|
| 269 |
+
"learning_rate": 9.499999999999999e-07,
|
| 270 |
+
"loss": -0.1421,
|
| 271 |
+
"num_tokens": 12279580.0,
|
| 272 |
+
"reward": -0.31881226040422916,
|
| 273 |
+
"reward_std": 0.6840411424636841,
|
| 274 |
+
"rewards/cosine_scaled_reward": -0.32439261162653565,
|
| 275 |
+
"rewards/format_reward": 0.005580357392318547,
|
| 276 |
+
"step": 19
|
| 277 |
+
},
|
| 278 |
+
{
|
| 279 |
+
"clip_ratio": 0.0,
|
| 280 |
+
"completion_length": 647.7310485839844,
|
| 281 |
+
"epoch": 0.20278833967046894,
|
| 282 |
+
"grad_norm": 0.1795511543750763,
|
| 283 |
+
"learning_rate": 1e-06,
|
| 284 |
+
"loss": -0.0997,
|
| 285 |
+
"num_tokens": 12980371.0,
|
| 286 |
+
"reward": -0.29815168026834726,
|
| 287 |
+
"reward_std": 0.6471684239804745,
|
| 288 |
+
"rewards/cosine_scaled_reward": -0.30038381088525057,
|
| 289 |
+
"rewards/format_reward": 0.0022321429569274187,
|
| 290 |
+
"step": 20
|
| 291 |
+
},
|
| 292 |
+
{
|
| 293 |
+
"clip_ratio": 0.0,
|
| 294 |
+
"completion_length": 660.1004867553711,
|
| 295 |
+
"epoch": 0.21292775665399238,
|
| 296 |
+
"grad_norm": 0.1688115894794464,
|
| 297 |
+
"learning_rate": 9.999203468625015e-07,
|
| 298 |
+
"loss": -0.0604,
|
| 299 |
+
"num_tokens": 13693093.0,
|
| 300 |
+
"reward": -0.11793683946598321,
|
| 301 |
+
"reward_std": 0.6547496430575848,
|
| 302 |
+
"rewards/cosine_scaled_reward": -0.11905290477443486,
|
| 303 |
+
"rewards/format_reward": 0.0011160714784637094,
|
| 304 |
+
"step": 21
|
| 305 |
+
},
|
| 306 |
+
{
|
| 307 |
+
"clip_ratio": 0.0,
|
| 308 |
+
"completion_length": 716.8471298217773,
|
| 309 |
+
"epoch": 0.22306717363751585,
|
| 310 |
+
"grad_norm": 0.1519731730222702,
|
| 311 |
+
"learning_rate": 9.99681412828496e-07,
|
| 312 |
+
"loss": -0.0818,
|
| 313 |
+
"num_tokens": 14468548.0,
|
| 314 |
+
"reward": -0.2080770255997777,
|
| 315 |
+
"reward_std": 0.5903773456811905,
|
| 316 |
+
"rewards/cosine_scaled_reward": -0.21030915481969714,
|
| 317 |
+
"rewards/format_reward": 0.0022321429569274187,
|
| 318 |
+
"step": 22
|
| 319 |
+
},
|
| 320 |
+
{
|
| 321 |
+
"clip_ratio": 0.0,
|
| 322 |
+
"completion_length": 675.1094131469727,
|
| 323 |
+
"epoch": 0.2332065906210393,
|
| 324 |
+
"grad_norm": 0.16422481834888458,
|
| 325 |
+
"learning_rate": 9.992832740253644e-07,
|
| 326 |
+
"loss": -0.0618,
|
| 327 |
+
"num_tokens": 15211694.0,
|
| 328 |
+
"reward": -0.13133149535860866,
|
| 329 |
+
"reward_std": 0.6078522019088268,
|
| 330 |
+
"rewards/cosine_scaled_reward": -0.1324475615983829,
|
| 331 |
+
"rewards/format_reward": 0.0011160714784637094,
|
| 332 |
+
"step": 23
|
| 333 |
+
},
|
| 334 |
+
{
|
| 335 |
+
"clip_ratio": 0.0,
|
| 336 |
+
"completion_length": 732.0714569091797,
|
| 337 |
+
"epoch": 0.24334600760456274,
|
| 338 |
+
"grad_norm": 0.14371691644191742,
|
| 339 |
+
"learning_rate": 9.987260573051267e-07,
|
| 340 |
+
"loss": -0.0698,
|
| 341 |
+
"num_tokens": 15997174.0,
|
| 342 |
+
"reward": -0.12170885177329183,
|
| 343 |
+
"reward_std": 0.5266584008932114,
|
| 344 |
+
"rewards/cosine_scaled_reward": -0.12282492127269506,
|
| 345 |
+
"rewards/format_reward": 0.0011160714784637094,
|
| 346 |
+
"step": 24
|
| 347 |
+
},
|
| 348 |
+
{
|
| 349 |
+
"clip_ratio": 0.0,
|
| 350 |
+
"completion_length": 674.2221298217773,
|
| 351 |
+
"epoch": 0.2534854245880862,
|
| 352 |
+
"grad_norm": 0.4856448471546173,
|
| 353 |
+
"learning_rate": 9.98009940204023e-07,
|
| 354 |
+
"loss": -0.047,
|
| 355 |
+
"num_tokens": 16732605.0,
|
| 356 |
+
"reward": 0.00843854621052742,
|
| 357 |
+
"reward_std": 0.524466548115015,
|
| 358 |
+
"rewards/cosine_scaled_reward": 0.00843854807317257,
|
| 359 |
+
"rewards/format_reward": 0.0,
|
| 360 |
+
"step": 25
|
| 361 |
+
},
|
| 362 |
+
{
|
| 363 |
+
"clip_ratio": 0.0,
|
| 364 |
+
"completion_length": 704.0792770385742,
|
| 365 |
+
"epoch": 0.26362484157160965,
|
| 366 |
+
"grad_norm": 0.15166230499744415,
|
| 367 |
+
"learning_rate": 9.971351508859486e-07,
|
| 368 |
+
"loss": -0.0499,
|
| 369 |
+
"num_tokens": 17482292.0,
|
| 370 |
+
"reward": 0.026816920144483447,
|
| 371 |
+
"reward_std": 0.5594276264309883,
|
| 372 |
+
"rewards/cosine_scaled_reward": 0.02570084948092699,
|
| 373 |
+
"rewards/format_reward": 0.0011160714784637094,
|
| 374 |
+
"step": 26
|
| 375 |
+
},
|
| 376 |
+
{
|
| 377 |
+
"clip_ratio": 0.0,
|
| 378 |
+
"completion_length": 676.5524826049805,
|
| 379 |
+
"epoch": 0.2737642585551331,
|
| 380 |
+
"grad_norm": 0.2196025401353836,
|
| 381 |
+
"learning_rate": 9.961019680697591e-07,
|
| 382 |
+
"loss": -0.0143,
|
| 383 |
+
"num_tokens": 18216659.0,
|
| 384 |
+
"reward": -0.00026240432634949684,
|
| 385 |
+
"reward_std": 0.587770089507103,
|
| 386 |
+
"rewards/cosine_scaled_reward": -0.0002624071203172207,
|
| 387 |
+
"rewards/format_reward": 0.0,
|
| 388 |
+
"step": 27
|
| 389 |
+
},
|
| 390 |
+
{
|
| 391 |
+
"clip_ratio": 0.0,
|
| 392 |
+
"completion_length": 751.2332916259766,
|
| 393 |
+
"epoch": 0.28390367553865653,
|
| 394 |
+
"grad_norm": 1.717673420906067,
|
| 395 |
+
"learning_rate": 9.949107209404663e-07,
|
| 396 |
+
"loss": 0.0007,
|
| 397 |
+
"num_tokens": 19023404.0,
|
| 398 |
+
"reward": 0.06609318475238979,
|
| 399 |
+
"reward_std": 0.5609292760491371,
|
| 400 |
+
"rewards/cosine_scaled_reward": 0.06609318096889183,
|
| 401 |
+
"rewards/format_reward": 0.0,
|
| 402 |
+
"step": 28
|
| 403 |
+
},
|
| 404 |
+
{
|
| 405 |
+
"clip_ratio": 0.0,
|
| 406 |
+
"completion_length": 717.3995819091797,
|
| 407 |
+
"epoch": 0.29404309252217997,
|
| 408 |
+
"grad_norm": 0.1441808044910431,
|
| 409 |
+
"learning_rate": 9.935617890443554e-07,
|
| 410 |
+
"loss": -0.0205,
|
| 411 |
+
"num_tokens": 19806930.0,
|
| 412 |
+
"reward": 0.054041724652051926,
|
| 413 |
+
"reward_std": 0.5101928748190403,
|
| 414 |
+
"rewards/cosine_scaled_reward": 0.05404172372072935,
|
| 415 |
+
"rewards/format_reward": 0.0,
|
| 416 |
+
"step": 29
|
| 417 |
+
},
|
| 418 |
+
{
|
| 419 |
+
"clip_ratio": 0.0,
|
| 420 |
+
"completion_length": 805.4955749511719,
|
| 421 |
+
"epoch": 0.3041825095057034,
|
| 422 |
+
"grad_norm": 0.12774203717708588,
|
| 423 |
+
"learning_rate": 9.92055602168058e-07,
|
| 424 |
+
"loss": -0.007,
|
| 425 |
+
"num_tokens": 20674142.0,
|
| 426 |
+
"reward": 0.047311737202107906,
|
| 427 |
+
"reward_std": 0.4787377752363682,
|
| 428 |
+
"rewards/cosine_scaled_reward": 0.04731173254549503,
|
| 429 |
+
"rewards/format_reward": 0.0,
|
| 430 |
+
"step": 30
|
| 431 |
+
},
|
| 432 |
+
{
|
| 433 |
+
"clip_ratio": 0.0,
|
| 434 |
+
"completion_length": 771.9632110595703,
|
| 435 |
+
"epoch": 0.31432192648922685,
|
| 436 |
+
"grad_norm": 0.1265403777360916,
|
| 437 |
+
"learning_rate": 9.90392640201615e-07,
|
| 438 |
+
"loss": 0.0387,
|
| 439 |
+
"num_tokens": 21493429.0,
|
| 440 |
+
"reward": 0.13890161426388659,
|
| 441 |
+
"reward_std": 0.52209198102355,
|
| 442 |
+
"rewards/cosine_scaled_reward": 0.1389016114990227,
|
| 443 |
+
"rewards/format_reward": 0.0,
|
| 444 |
+
"step": 31
|
| 445 |
+
},
|
| 446 |
+
{
|
| 447 |
+
"clip_ratio": 0.0,
|
| 448 |
+
"completion_length": 763.1551666259766,
|
| 449 |
+
"epoch": 0.3244613434727503,
|
| 450 |
+
"grad_norm": 0.7253857851028442,
|
| 451 |
+
"learning_rate": 9.885734329855797e-07,
|
| 452 |
+
"loss": 0.023,
|
| 453 |
+
"num_tokens": 22296960.0,
|
| 454 |
+
"reward": 0.20044966787099838,
|
| 455 |
+
"reward_std": 0.4556784965097904,
|
| 456 |
+
"rewards/cosine_scaled_reward": 0.20044966042041779,
|
| 457 |
+
"rewards/format_reward": 0.0,
|
| 458 |
+
"step": 32
|
| 459 |
+
},
|
| 460 |
+
{
|
| 461 |
+
"clip_ratio": 0.0,
|
| 462 |
+
"completion_length": 778.8873138427734,
|
| 463 |
+
"epoch": 0.33460076045627374,
|
| 464 |
+
"grad_norm": 0.1363135427236557,
|
| 465 |
+
"learning_rate": 9.865985601422017e-07,
|
| 466 |
+
"loss": -0.0147,
|
| 467 |
+
"num_tokens": 23128899.0,
|
| 468 |
+
"reward": 0.05959217494819313,
|
| 469 |
+
"reward_std": 0.4793633334338665,
|
| 470 |
+
"rewards/cosine_scaled_reward": 0.059291763755027205,
|
| 471 |
+
"rewards/format_reward": 0.0,
|
| 472 |
+
"step": 33
|
| 473 |
+
},
|
| 474 |
+
{
|
| 475 |
+
"clip_ratio": 0.0,
|
| 476 |
+
"completion_length": 773.6607513427734,
|
| 477 |
+
"epoch": 0.34474017743979724,
|
| 478 |
+
"grad_norm": 0.12697336077690125,
|
| 479 |
+
"learning_rate": 9.844686508907537e-07,
|
| 480 |
+
"loss": 0.0046,
|
| 481 |
+
"num_tokens": 23949587.0,
|
| 482 |
+
"reward": 0.15802541188895702,
|
| 483 |
+
"reward_std": 0.4121544286608696,
|
| 484 |
+
"rewards/cosine_scaled_reward": 0.15802540606819093,
|
| 485 |
+
"rewards/format_reward": 0.0,
|
| 486 |
+
"step": 34
|
| 487 |
+
},
|
| 488 |
+
{
|
| 489 |
+
"clip_ratio": 0.0,
|
| 490 |
+
"completion_length": 755.8393173217773,
|
| 491 |
+
"epoch": 0.3548795944233207,
|
| 492 |
+
"grad_norm": 0.13817483186721802,
|
| 493 |
+
"learning_rate": 9.821843838470534e-07,
|
| 494 |
+
"loss": 0.002,
|
| 495 |
+
"num_tokens": 24769547.0,
|
| 496 |
+
"reward": 0.15929587977007031,
|
| 497 |
+
"reward_std": 0.4162367284297943,
|
| 498 |
+
"rewards/cosine_scaled_reward": 0.159295872785151,
|
| 499 |
+
"rewards/format_reward": 0.0,
|
| 500 |
+
"step": 35
|
| 501 |
+
},
|
| 502 |
+
{
|
| 503 |
+
"clip_ratio": 0.0,
|
| 504 |
+
"completion_length": 758.3839645385742,
|
| 505 |
+
"epoch": 0.3650190114068441,
|
| 506 |
+
"grad_norm": 0.1282864660024643,
|
| 507 |
+
"learning_rate": 9.797464868072486e-07,
|
| 508 |
+
"loss": 0.0237,
|
| 509 |
+
"num_tokens": 25577171.0,
|
| 510 |
+
"reward": 0.193751014303416,
|
| 511 |
+
"reward_std": 0.48775994032621384,
|
| 512 |
+
"rewards/cosine_scaled_reward": 0.193751011043787,
|
| 513 |
+
"rewards/format_reward": 0.0,
|
| 514 |
+
"step": 36
|
| 515 |
+
},
|
| 516 |
+
{
|
| 517 |
+
"clip_ratio": 0.0,
|
| 518 |
+
"completion_length": 737.6127624511719,
|
| 519 |
+
"epoch": 0.37515842839036756,
|
| 520 |
+
"grad_norm": 0.13303731381893158,
|
| 521 |
+
"learning_rate": 9.771557365159319e-07,
|
| 522 |
+
"loss": 0.0083,
|
| 523 |
+
"num_tokens": 26363600.0,
|
| 524 |
+
"reward": 0.13768478552810848,
|
| 525 |
+
"reward_std": 0.40786123648285866,
|
| 526 |
+
"rewards/cosine_scaled_reward": 0.1376847765641287,
|
| 527 |
+
"rewards/format_reward": 0.0,
|
| 528 |
+
"step": 37
|
| 529 |
+
},
|
| 530 |
+
{
|
| 531 |
+
"clip_ratio": 0.0,
|
| 532 |
+
"completion_length": 753.794677734375,
|
| 533 |
+
"epoch": 0.385297845373891,
|
| 534 |
+
"grad_norm": 0.137796089053154,
|
| 535 |
+
"learning_rate": 9.744129584186597e-07,
|
| 536 |
+
"loss": 0.0127,
|
| 537 |
+
"num_tokens": 27165608.0,
|
| 538 |
+
"reward": 0.12427436339203268,
|
| 539 |
+
"reward_std": 0.47414594888687134,
|
| 540 |
+
"rewards/cosine_scaled_reward": 0.12427435873541981,
|
| 541 |
+
"rewards/format_reward": 0.0,
|
| 542 |
+
"step": 38
|
| 543 |
+
},
|
| 544 |
+
{
|
| 545 |
+
"clip_ratio": 0.0,
|
| 546 |
+
"completion_length": 799.005615234375,
|
| 547 |
+
"epoch": 0.39543726235741444,
|
| 548 |
+
"grad_norm": 0.1327836662530899,
|
| 549 |
+
"learning_rate": 9.71519026398956e-07,
|
| 550 |
+
"loss": 0.0341,
|
| 551 |
+
"num_tokens": 28019301.0,
|
| 552 |
+
"reward": 0.12463877908885479,
|
| 553 |
+
"reward_std": 0.37975190207362175,
|
| 554 |
+
"rewards/cosine_scaled_reward": 0.12463877722620964,
|
| 555 |
+
"rewards/format_reward": 0.0,
|
| 556 |
+
"step": 39
|
| 557 |
+
},
|
| 558 |
+
{
|
| 559 |
+
"clip_ratio": 0.0,
|
| 560 |
+
"completion_length": 732.2611846923828,
|
| 561 |
+
"epoch": 0.4055766793409379,
|
| 562 |
+
"grad_norm": 0.14108538627624512,
|
| 563 |
+
"learning_rate": 9.68474862499881e-07,
|
| 564 |
+
"loss": 0.0519,
|
| 565 |
+
"num_tokens": 28794671.0,
|
| 566 |
+
"reward": 0.17419127095490694,
|
| 567 |
+
"reward_std": 0.44464627653360367,
|
| 568 |
+
"rewards/cosine_scaled_reward": 0.17419126071035862,
|
| 569 |
+
"rewards/format_reward": 0.0,
|
| 570 |
+
"step": 40
|
| 571 |
+
},
|
| 572 |
+
{
|
| 573 |
+
"clip_ratio": 0.0,
|
| 574 |
+
"completion_length": 725.4620819091797,
|
| 575 |
+
"epoch": 0.4157160963244613,
|
| 576 |
+
"grad_norm": 0.13821274042129517,
|
| 577 |
+
"learning_rate": 9.652814366302568e-07,
|
| 578 |
+
"loss": 0.0465,
|
| 579 |
+
"num_tokens": 29570661.0,
|
| 580 |
+
"reward": 0.32581639010459185,
|
| 581 |
+
"reward_std": 0.3625021446496248,
|
| 582 |
+
"rewards/cosine_scaled_reward": 0.325816385447979,
|
| 583 |
+
"rewards/format_reward": 0.0,
|
| 584 |
+
"step": 41
|
| 585 |
+
},
|
| 586 |
+
{
|
| 587 |
+
"clip_ratio": 0.0,
|
| 588 |
+
"completion_length": 730.3828430175781,
|
| 589 |
+
"epoch": 0.42585551330798477,
|
| 590 |
+
"grad_norm": 0.1342957466840744,
|
| 591 |
+
"learning_rate": 9.619397662556433e-07,
|
| 592 |
+
"loss": 0.0505,
|
| 593 |
+
"num_tokens": 30354052.0,
|
| 594 |
+
"reward": 0.23176710167899728,
|
| 595 |
+
"reward_std": 0.36819201707839966,
|
| 596 |
+
"rewards/cosine_scaled_reward": 0.23176709050312638,
|
| 597 |
+
"rewards/format_reward": 0.0,
|
| 598 |
+
"step": 42
|
| 599 |
+
},
|
| 600 |
+
{
|
| 601 |
+
"clip_ratio": 0.0,
|
| 602 |
+
"completion_length": 795.7176742553711,
|
| 603 |
+
"epoch": 0.43599493029150826,
|
| 604 |
+
"grad_norm": 0.12134254723787308,
|
| 605 |
+
"learning_rate": 9.5845091607416e-07,
|
| 606 |
+
"loss": 0.041,
|
| 607 |
+
"num_tokens": 31185719.0,
|
| 608 |
+
"reward": 0.2366194719215855,
|
| 609 |
+
"reward_std": 0.4014102444052696,
|
| 610 |
+
"rewards/cosine_scaled_reward": 0.23661946202628314,
|
| 611 |
+
"rewards/format_reward": 0.0,
|
| 612 |
+
"step": 43
|
| 613 |
+
},
|
| 614 |
+
{
|
| 615 |
+
"clip_ratio": 0.0,
|
| 616 |
+
"completion_length": 801.1931076049805,
|
| 617 |
+
"epoch": 0.4461343472750317,
|
| 618 |
+
"grad_norm": 0.7807191014289856,
|
| 619 |
+
"learning_rate": 9.548159976772592e-07,
|
| 620 |
+
"loss": 0.0134,
|
| 621 |
+
"num_tokens": 32033636.0,
|
| 622 |
+
"reward": 0.21036180562805384,
|
| 623 |
+
"reward_std": 0.40362199395895004,
|
| 624 |
+
"rewards/cosine_scaled_reward": 0.21036179037764668,
|
| 625 |
+
"rewards/format_reward": 0.0,
|
| 626 |
+
"step": 44
|
| 627 |
+
},
|
| 628 |
+
{
|
| 629 |
+
"clip_ratio": 0.0,
|
| 630 |
+
"completion_length": 819.1339569091797,
|
| 631 |
+
"epoch": 0.45627376425855515,
|
| 632 |
+
"grad_norm": 0.12294432520866394,
|
| 633 |
+
"learning_rate": 9.510361691955606e-07,
|
| 634 |
+
"loss": 0.021,
|
| 635 |
+
"num_tokens": 32890028.0,
|
| 636 |
+
"reward": 0.12139649732853286,
|
| 637 |
+
"reward_std": 0.4755956120789051,
|
| 638 |
+
"rewards/cosine_scaled_reward": 0.12139649360324256,
|
| 639 |
+
"rewards/format_reward": 0.0,
|
| 640 |
+
"step": 45
|
| 641 |
+
},
|
| 642 |
+
{
|
| 643 |
+
"clip_ratio": 0.0,
|
| 644 |
+
"completion_length": 717.7433319091797,
|
| 645 |
+
"epoch": 0.4664131812420786,
|
| 646 |
+
"grad_norm": 0.3100641071796417,
|
| 647 |
+
"learning_rate": 9.471126349298556e-07,
|
| 648 |
+
"loss": 0.0361,
|
| 649 |
+
"num_tokens": 33663342.0,
|
| 650 |
+
"reward": 0.23560622334480286,
|
| 651 |
+
"reward_std": 0.42723193392157555,
|
| 652 |
+
"rewards/cosine_scaled_reward": 0.23560621310025454,
|
| 653 |
+
"rewards/format_reward": 0.0,
|
| 654 |
+
"step": 46
|
| 655 |
+
},
|
| 656 |
+
{
|
| 657 |
+
"clip_ratio": 0.0,
|
| 658 |
+
"completion_length": 762.2913284301758,
|
| 659 |
+
"epoch": 0.47655259822560203,
|
| 660 |
+
"grad_norm": 0.12677805125713348,
|
| 661 |
+
"learning_rate": 9.430466449674013e-07,
|
| 662 |
+
"loss": -0.0045,
|
| 663 |
+
"num_tokens": 34494707.0,
|
| 664 |
+
"reward": 0.1484816954471171,
|
| 665 |
+
"reward_std": 0.3901702091097832,
|
| 666 |
+
"rewards/cosine_scaled_reward": 0.14848168776370585,
|
| 667 |
+
"rewards/format_reward": 0.0,
|
| 668 |
+
"step": 47
|
| 669 |
+
},
|
| 670 |
+
{
|
| 671 |
+
"clip_ratio": 0.0,
|
| 672 |
+
"completion_length": 789.130615234375,
|
| 673 |
+
"epoch": 0.4866920152091255,
|
| 674 |
+
"grad_norm": 0.12980540096759796,
|
| 675 |
+
"learning_rate": 9.388394947836278e-07,
|
| 676 |
+
"loss": -0.008,
|
| 677 |
+
"num_tokens": 35328504.0,
|
| 678 |
+
"reward": 0.09070475457701832,
|
| 679 |
+
"reward_std": 0.3948558494448662,
|
| 680 |
+
"rewards/cosine_scaled_reward": 0.09070474898908287,
|
| 681 |
+
"rewards/format_reward": 0.0,
|
| 682 |
+
"step": 48
|
| 683 |
+
},
|
| 684 |
+
{
|
| 685 |
+
"clip_ratio": 0.0,
|
| 686 |
+
"completion_length": 792.5145416259766,
|
| 687 |
+
"epoch": 0.4968314321926489,
|
| 688 |
+
"grad_norm": 0.1316457837820053,
|
| 689 |
+
"learning_rate": 9.344925248293835e-07,
|
| 690 |
+
"loss": 0.0133,
|
| 691 |
+
"num_tokens": 36159605.0,
|
| 692 |
+
"reward": 0.2209687788854353,
|
| 693 |
+
"reward_std": 0.39467399939894676,
|
| 694 |
+
"rewards/cosine_scaled_reward": 0.22096876305295154,
|
| 695 |
+
"rewards/format_reward": 0.0,
|
| 696 |
+
"step": 49
|
| 697 |
+
},
|
| 698 |
+
{
|
| 699 |
+
"clip_ratio": 0.0,
|
| 700 |
+
"completion_length": 776.4721374511719,
|
| 701 |
+
"epoch": 0.5069708491761724,
|
| 702 |
+
"grad_norm": 0.12961426377296448,
|
| 703 |
+
"learning_rate": 9.300071201038501e-07,
|
| 704 |
+
"loss": -0.0046,
|
| 705 |
+
"num_tokens": 36981500.0,
|
| 706 |
+
"reward": 0.15061184321530163,
|
| 707 |
+
"reward_std": 0.4538601525127888,
|
| 708 |
+
"rewards/cosine_scaled_reward": 0.15061183541547507,
|
| 709 |
+
"rewards/format_reward": 0.0,
|
| 710 |
+
"step": 50
|
| 711 |
+
},
|
| 712 |
+
{
|
| 713 |
+
"clip_ratio": 0.0,
|
| 714 |
+
"completion_length": 665.3649826049805,
|
| 715 |
+
"epoch": 0.5171102661596958,
|
| 716 |
+
"grad_norm": 0.1373734474182129,
|
| 717 |
+
"learning_rate": 9.253847097132655e-07,
|
| 718 |
+
"loss": 0.0202,
|
| 719 |
+
"num_tokens": 37708491.0,
|
| 720 |
+
"reward": 0.25066345604136586,
|
| 721 |
+
"reward_std": 0.41370486095547676,
|
| 722 |
+
"rewards/cosine_scaled_reward": 0.25066344952210784,
|
| 723 |
+
"rewards/format_reward": 0.0,
|
| 724 |
+
"step": 51
|
| 725 |
+
},
|
| 726 |
+
{
|
| 727 |
+
"clip_ratio": 0.0,
|
| 728 |
+
"completion_length": 721.5212249755859,
|
| 729 |
+
"epoch": 0.5272496831432193,
|
| 730 |
+
"grad_norm": 0.12883824110031128,
|
| 731 |
+
"learning_rate": 9.206267664155906e-07,
|
| 732 |
+
"loss": 0.0295,
|
| 733 |
+
"num_tokens": 38475350.0,
|
| 734 |
+
"reward": 0.2094368627294898,
|
| 735 |
+
"reward_std": 0.4836310148239136,
|
| 736 |
+
"rewards/cosine_scaled_reward": 0.2094368515536189,
|
| 737 |
+
"rewards/format_reward": 0.0,
|
| 738 |
+
"step": 52
|
| 739 |
+
},
|
| 740 |
+
{
|
| 741 |
+
"clip_ratio": 0.0,
|
| 742 |
+
"completion_length": 748.3538284301758,
|
| 743 |
+
"epoch": 0.5373891001267427,
|
| 744 |
+
"grad_norm": 0.1301122009754181,
|
| 745 |
+
"learning_rate": 9.157348061512726e-07,
|
| 746 |
+
"loss": 0.0326,
|
| 747 |
+
"num_tokens": 39281955.0,
|
| 748 |
+
"reward": 0.19618262629956007,
|
| 749 |
+
"reward_std": 0.3902331814169884,
|
| 750 |
+
"rewards/cosine_scaled_reward": 0.1961826179176569,
|
| 751 |
+
"rewards/format_reward": 0.0,
|
| 752 |
+
"step": 53
|
| 753 |
+
},
|
| 754 |
+
{
|
| 755 |
+
"clip_ratio": 0.0,
|
| 756 |
+
"completion_length": 820.4486923217773,
|
| 757 |
+
"epoch": 0.5475285171102662,
|
| 758 |
+
"grad_norm": 0.12074018269777298,
|
| 759 |
+
"learning_rate": 9.107103875602458e-07,
|
| 760 |
+
"loss": -0.0192,
|
| 761 |
+
"num_tokens": 40135629.0,
|
| 762 |
+
"reward": 0.1222395608201623,
|
| 763 |
+
"reward_std": 0.4181748628616333,
|
| 764 |
+
"rewards/cosine_scaled_reward": 0.12223955010995269,
|
| 765 |
+
"rewards/format_reward": 0.0,
|
| 766 |
+
"step": 54
|
| 767 |
+
},
|
| 768 |
+
{
|
| 769 |
+
"clip_ratio": 0.0,
|
| 770 |
+
"completion_length": 758.4922256469727,
|
| 771 |
+
"epoch": 0.5576679340937896,
|
| 772 |
+
"grad_norm": 0.14272169768810272,
|
| 773 |
+
"learning_rate": 9.055551114853295e-07,
|
| 774 |
+
"loss": 0.0269,
|
| 775 |
+
"num_tokens": 40945630.0,
|
| 776 |
+
"reward": 0.1248078744392842,
|
| 777 |
+
"reward_std": 0.445852842181921,
|
| 778 |
+
"rewards/cosine_scaled_reward": 0.12480786850210279,
|
| 779 |
+
"rewards/format_reward": 0.0,
|
| 780 |
+
"step": 55
|
| 781 |
+
},
|
| 782 |
+
{
|
| 783 |
+
"clip_ratio": 0.0,
|
| 784 |
+
"completion_length": 661.1495895385742,
|
| 785 |
+
"epoch": 0.5678073510773131,
|
| 786 |
+
"grad_norm": 0.2661519944667816,
|
| 787 |
+
"learning_rate": 9.002706204621802e-07,
|
| 788 |
+
"loss": 0.0013,
|
| 789 |
+
"num_tokens": 41661700.0,
|
| 790 |
+
"reward": 0.21717188507318497,
|
| 791 |
+
"reward_std": 0.46877430006861687,
|
| 792 |
+
"rewards/cosine_scaled_reward": 0.21717186993919313,
|
| 793 |
+
"rewards/format_reward": 0.0,
|
| 794 |
+
"step": 56
|
| 795 |
+
},
|
| 796 |
+
{
|
| 797 |
+
"clip_ratio": 0.0,
|
| 798 |
+
"completion_length": 708.866096496582,
|
| 799 |
+
"epoch": 0.5779467680608364,
|
| 800 |
+
"grad_norm": 0.15228039026260376,
|
| 801 |
+
"learning_rate": 8.948585981959578e-07,
|
| 802 |
+
"loss": 0.0119,
|
| 803 |
+
"num_tokens": 42428428.0,
|
| 804 |
+
"reward": 0.25196343823336065,
|
| 805 |
+
"reward_std": 0.3885682690888643,
|
| 806 |
+
"rewards/cosine_scaled_reward": 0.2519634268246591,
|
| 807 |
+
"rewards/format_reward": 0.0,
|
| 808 |
+
"step": 57
|
| 809 |
+
},
|
| 810 |
+
{
|
| 811 |
+
"clip_ratio": 0.0,
|
| 812 |
+
"completion_length": 707.9698944091797,
|
| 813 |
+
"epoch": 0.5880861850443599,
|
| 814 |
+
"grad_norm": 0.13880610466003418,
|
| 815 |
+
"learning_rate": 8.893207690248775e-07,
|
| 816 |
+
"loss": 0.0344,
|
| 817 |
+
"num_tokens": 43183833.0,
|
| 818 |
+
"reward": 0.22749723307788372,
|
| 819 |
+
"reward_std": 0.41020799428224564,
|
| 820 |
+
"rewards/cosine_scaled_reward": 0.22749722423031926,
|
| 821 |
+
"rewards/format_reward": 0.0,
|
| 822 |
+
"step": 58
|
| 823 |
+
},
|
| 824 |
+
{
|
| 825 |
+
"clip_ratio": 0.0,
|
| 826 |
+
"completion_length": 698.7991333007812,
|
| 827 |
+
"epoch": 0.5982256020278834,
|
| 828 |
+
"grad_norm": 0.14641599357128143,
|
| 829 |
+
"learning_rate": 8.836588973708128e-07,
|
| 830 |
+
"loss": -0.0323,
|
| 831 |
+
"num_tokens": 43951549.0,
|
| 832 |
+
"reward": 0.11083738692104816,
|
| 833 |
+
"reward_std": 0.45132191106677055,
|
| 834 |
+
"rewards/cosine_scaled_reward": 0.11083737714216113,
|
| 835 |
+
"rewards/format_reward": 0.0,
|
| 836 |
+
"step": 59
|
| 837 |
+
},
|
| 838 |
+
{
|
| 839 |
+
"clip_ratio": 0.0,
|
| 840 |
+
"completion_length": 749.3850860595703,
|
| 841 |
+
"epoch": 0.6083650190114068,
|
| 842 |
+
"grad_norm": 0.14265136420726776,
|
| 843 |
+
"learning_rate": 8.778747871771291e-07,
|
| 844 |
+
"loss": 0.0228,
|
| 845 |
+
"num_tokens": 44760246.0,
|
| 846 |
+
"reward": 0.15983119537122548,
|
| 847 |
+
"reward_std": 0.4386202208697796,
|
| 848 |
+
"rewards/cosine_scaled_reward": 0.15983118978329003,
|
| 849 |
+
"rewards/format_reward": 0.0,
|
| 850 |
+
"step": 60
|
| 851 |
+
},
|
| 852 |
+
{
|
| 853 |
+
"clip_ratio": 0.0,
|
| 854 |
+
"completion_length": 700.7500305175781,
|
| 855 |
+
"epoch": 0.6185044359949303,
|
| 856 |
+
"grad_norm": 0.131936714053154,
|
| 857 |
+
"learning_rate": 8.719702813339247e-07,
|
| 858 |
+
"loss": 0.0532,
|
| 859 |
+
"num_tokens": 45510462.0,
|
| 860 |
+
"reward": 0.23273684654850513,
|
| 861 |
+
"reward_std": 0.4278288222849369,
|
| 862 |
+
"rewards/cosine_scaled_reward": 0.23273683816660196,
|
| 863 |
+
"rewards/format_reward": 0.0,
|
| 864 |
+
"step": 61
|
| 865 |
+
},
|
| 866 |
+
{
|
| 867 |
+
"clip_ratio": 0.0,
|
| 868 |
+
"completion_length": 729.8102951049805,
|
| 869 |
+
"epoch": 0.6286438529784537,
|
| 870 |
+
"grad_norm": 0.13723866641521454,
|
| 871 |
+
"learning_rate": 8.659472610908627e-07,
|
| 872 |
+
"loss": -0.0047,
|
| 873 |
+
"num_tokens": 46293764.0,
|
| 874 |
+
"reward": 0.15899321692995727,
|
| 875 |
+
"reward_std": 0.3875345978885889,
|
| 876 |
+
"rewards/cosine_scaled_reward": 0.15899321576580405,
|
| 877 |
+
"rewards/format_reward": 0.0,
|
| 878 |
+
"step": 62
|
| 879 |
+
},
|
| 880 |
+
{
|
| 881 |
+
"clip_ratio": 0.0,
|
| 882 |
+
"completion_length": 631.8594055175781,
|
| 883 |
+
"epoch": 0.6387832699619772,
|
| 884 |
+
"grad_norm": 0.15783964097499847,
|
| 885 |
+
"learning_rate": 8.598076454577814e-07,
|
| 886 |
+
"loss": 0.0349,
|
| 887 |
+
"num_tokens": 46984206.0,
|
| 888 |
+
"reward": 0.28133094896293187,
|
| 889 |
+
"reward_std": 0.4228518716990948,
|
| 890 |
+
"rewards/cosine_scaled_reward": 0.2813309368539194,
|
| 891 |
+
"rewards/format_reward": 0.0,
|
| 892 |
+
"step": 63
|
| 893 |
+
},
|
| 894 |
+
{
|
| 895 |
+
"clip_ratio": 0.0,
|
| 896 |
+
"completion_length": 649.584846496582,
|
| 897 |
+
"epoch": 0.6489226869455006,
|
| 898 |
+
"grad_norm": 0.15222658216953278,
|
| 899 |
+
"learning_rate": 8.535533905932737e-07,
|
| 900 |
+
"loss": 0.0307,
|
| 901 |
+
"num_tokens": 47714362.0,
|
| 902 |
+
"reward": 0.28929764311760664,
|
| 903 |
+
"reward_std": 0.3426767401397228,
|
| 904 |
+
"rewards/cosine_scaled_reward": 0.2892976254224777,
|
| 905 |
+
"rewards/format_reward": 0.0,
|
| 906 |
+
"step": 64
|
| 907 |
+
},
|
| 908 |
+
{
|
| 909 |
+
"clip_ratio": 0.0,
|
| 910 |
+
"completion_length": 679.0524826049805,
|
| 911 |
+
"epoch": 0.6590621039290241,
|
| 912 |
+
"grad_norm": 0.17544390261173248,
|
| 913 |
+
"learning_rate": 8.471864891814304e-07,
|
| 914 |
+
"loss": 0.0596,
|
| 915 |
+
"num_tokens": 48441897.0,
|
| 916 |
+
"reward": 0.2452558323275298,
|
| 917 |
+
"reward_std": 0.4179961830377579,
|
| 918 |
+
"rewards/cosine_scaled_reward": 0.24525582510977983,
|
| 919 |
+
"rewards/format_reward": 0.0,
|
| 920 |
+
"step": 65
|
| 921 |
+
},
|
| 922 |
+
{
|
| 923 |
+
"clip_ratio": 0.0,
|
| 924 |
+
"completion_length": 734.794677734375,
|
| 925 |
+
"epoch": 0.6692015209125475,
|
| 926 |
+
"grad_norm": 0.13531185686588287,
|
| 927 |
+
"learning_rate": 8.407089697969456e-07,
|
| 928 |
+
"loss": 0.0013,
|
| 929 |
+
"num_tokens": 49225137.0,
|
| 930 |
+
"reward": 0.24510841415030882,
|
| 931 |
+
"reward_std": 0.40535635873675346,
|
| 932 |
+
"rewards/cosine_scaled_reward": 0.24510839313734323,
|
| 933 |
+
"rewards/format_reward": 0.0,
|
| 934 |
+
"step": 66
|
| 935 |
+
},
|
| 936 |
+
{
|
| 937 |
+
"clip_ratio": 0.0,
|
| 938 |
+
"completion_length": 763.2321701049805,
|
| 939 |
+
"epoch": 0.679340937896071,
|
| 940 |
+
"grad_norm": 0.1570795476436615,
|
| 941 |
+
"learning_rate": 8.341228962587881e-07,
|
| 942 |
+
"loss": 0.0186,
|
| 943 |
+
"num_tokens": 50045121.0,
|
| 944 |
+
"reward": 0.20169485360383987,
|
| 945 |
+
"reward_std": 0.37343652732670307,
|
| 946 |
+
"rewards/cosine_scaled_reward": 0.2016948414966464,
|
| 947 |
+
"rewards/format_reward": 0.0,
|
| 948 |
+
"step": 67
|
| 949 |
+
},
|
| 950 |
+
{
|
| 951 |
+
"clip_ratio": 0.0,
|
| 952 |
+
"completion_length": 628.7924346923828,
|
| 953 |
+
"epoch": 0.6894803548795945,
|
| 954 |
+
"grad_norm": 0.15324285626411438,
|
| 955 |
+
"learning_rate": 8.274303669726426e-07,
|
| 956 |
+
"loss": 0.0192,
|
| 957 |
+
"num_tokens": 50729839.0,
|
| 958 |
+
"reward": 0.32515534153208137,
|
| 959 |
+
"reward_std": 0.39401572197675705,
|
| 960 |
+
"rewards/cosine_scaled_reward": 0.32515532383695245,
|
| 961 |
+
"rewards/format_reward": 0.0,
|
| 962 |
+
"step": 68
|
| 963 |
+
},
|
| 964 |
+
{
|
| 965 |
+
"clip_ratio": 0.0,
|
| 966 |
+
"completion_length": 743.4654388427734,
|
| 967 |
+
"epoch": 0.6996197718631179,
|
| 968 |
+
"grad_norm": 0.14744603633880615,
|
| 969 |
+
"learning_rate": 8.206335142623304e-07,
|
| 970 |
+
"loss": -0.0119,
|
| 971 |
+
"num_tokens": 51525744.0,
|
| 972 |
+
"reward": 0.1557501032948494,
|
| 973 |
+
"reward_std": 0.4095492400228977,
|
| 974 |
+
"rewards/cosine_scaled_reward": 0.15575009072199464,
|
| 975 |
+
"rewards/format_reward": 0.0,
|
| 976 |
+
"step": 69
|
| 977 |
+
},
|
| 978 |
+
{
|
| 979 |
+
"clip_ratio": 0.0,
|
| 980 |
+
"completion_length": 700.7924575805664,
|
| 981 |
+
"epoch": 0.7097591888466414,
|
| 982 |
+
"grad_norm": 0.13947679102420807,
|
| 983 |
+
"learning_rate": 8.137345036904259e-07,
|
| 984 |
+
"loss": 0.0206,
|
| 985 |
+
"num_tokens": 52273030.0,
|
| 986 |
+
"reward": 0.14582239514857065,
|
| 987 |
+
"reward_std": 0.4498204067349434,
|
| 988 |
+
"rewards/cosine_scaled_reward": 0.14582238957518712,
|
| 989 |
+
"rewards/format_reward": 0.0,
|
| 990 |
+
"step": 70
|
| 991 |
+
},
|
| 992 |
+
{
|
| 993 |
+
"clip_ratio": 0.0,
|
| 994 |
+
"completion_length": 674.7065048217773,
|
| 995 |
+
"epoch": 0.7198986058301647,
|
| 996 |
+
"grad_norm": 0.14676493406295776,
|
| 997 |
+
"learning_rate": 8.067355333682797e-07,
|
| 998 |
+
"loss": 0.0221,
|
| 999 |
+
"num_tokens": 53016335.0,
|
| 1000 |
+
"reward": 0.18560006003826857,
|
| 1001 |
+
"reward_std": 0.39886176586151123,
|
| 1002 |
+
"rewards/cosine_scaled_reward": 0.1856000553816557,
|
| 1003 |
+
"rewards/format_reward": 0.0,
|
| 1004 |
+
"step": 71
|
| 1005 |
+
},
|
| 1006 |
+
{
|
| 1007 |
+
"clip_ratio": 0.0,
|
| 1008 |
+
"completion_length": 669.2221298217773,
|
| 1009 |
+
"epoch": 0.7300380228136882,
|
| 1010 |
+
"grad_norm": 0.13405963778495789,
|
| 1011 |
+
"learning_rate": 7.996388332556734e-07,
|
| 1012 |
+
"loss": 0.0196,
|
| 1013 |
+
"num_tokens": 53744678.0,
|
| 1014 |
+
"reward": 0.22391042113304138,
|
| 1015 |
+
"reward_std": 0.40190940350294113,
|
| 1016 |
+
"rewards/cosine_scaled_reward": 0.2239104132167995,
|
| 1017 |
+
"rewards/format_reward": 0.0,
|
| 1018 |
+
"step": 72
|
| 1019 |
+
},
|
| 1020 |
+
{
|
| 1021 |
+
"clip_ratio": 0.0,
|
| 1022 |
+
"completion_length": 718.1372909545898,
|
| 1023 |
+
"epoch": 0.7401774397972116,
|
| 1024 |
+
"grad_norm": 0.3063336908817291,
|
| 1025 |
+
"learning_rate": 7.924466644503264e-07,
|
| 1026 |
+
"loss": -0.0106,
|
| 1027 |
+
"num_tokens": 54520625.0,
|
| 1028 |
+
"reward": 0.20024659007322043,
|
| 1029 |
+
"reward_std": 0.42064137011766434,
|
| 1030 |
+
"rewards/cosine_scaled_reward": 0.2002465761033818,
|
| 1031 |
+
"rewards/format_reward": 0.0,
|
| 1032 |
+
"step": 73
|
| 1033 |
+
},
|
| 1034 |
+
{
|
| 1035 |
+
"clip_ratio": 0.0,
|
| 1036 |
+
"completion_length": 641.3616333007812,
|
| 1037 |
+
"epoch": 0.7503168567807351,
|
| 1038 |
+
"grad_norm": 0.16075171530246735,
|
| 1039 |
+
"learning_rate": 7.85161318467482e-07,
|
| 1040 |
+
"loss": 0.044,
|
| 1041 |
+
"num_tokens": 55230669.0,
|
| 1042 |
+
"reward": 0.2528219725936651,
|
| 1043 |
+
"reward_std": 0.4147798642516136,
|
| 1044 |
+
"rewards/cosine_scaled_reward": 0.2528219521045685,
|
| 1045 |
+
"rewards/format_reward": 0.0,
|
| 1046 |
+
"step": 74
|
| 1047 |
+
},
|
| 1048 |
+
{
|
| 1049 |
+
"clip_ratio": 0.0,
|
| 1050 |
+
"completion_length": 631.3672180175781,
|
| 1051 |
+
"epoch": 0.7604562737642585,
|
| 1052 |
+
"grad_norm": 0.6628883481025696,
|
| 1053 |
+
"learning_rate": 7.777851165098011e-07,
|
| 1054 |
+
"loss": 0.0325,
|
| 1055 |
+
"num_tokens": 55927054.0,
|
| 1056 |
+
"reward": 0.3146879682317376,
|
| 1057 |
+
"reward_std": 0.39327000454068184,
|
| 1058 |
+
"rewards/cosine_scaled_reward": 0.3146879549603909,
|
| 1059 |
+
"rewards/format_reward": 0.0,
|
| 1060 |
+
"step": 75
|
| 1061 |
+
},
|
| 1062 |
+
{
|
| 1063 |
+
"clip_ratio": 0.0,
|
| 1064 |
+
"completion_length": 611.7455635070801,
|
| 1065 |
+
"epoch": 0.770595690747782,
|
| 1066 |
+
"grad_norm": 0.15209974348545074,
|
| 1067 |
+
"learning_rate": 7.703204087277988e-07,
|
| 1068 |
+
"loss": 0.0055,
|
| 1069 |
+
"num_tokens": 56602418.0,
|
| 1070 |
+
"reward": 0.135146701708436,
|
| 1071 |
+
"reward_std": 0.4354245401918888,
|
| 1072 |
+
"rewards/cosine_scaled_reward": 0.13514669938012958,
|
| 1073 |
+
"rewards/format_reward": 0.0,
|
| 1074 |
+
"step": 76
|
| 1075 |
+
},
|
| 1076 |
+
{
|
| 1077 |
+
"clip_ratio": 0.0,
|
| 1078 |
+
"completion_length": 662.4486885070801,
|
| 1079 |
+
"epoch": 0.7807351077313055,
|
| 1080 |
+
"grad_norm": 0.14389804005622864,
|
| 1081 |
+
"learning_rate": 7.627695734710564e-07,
|
| 1082 |
+
"loss": -0.004,
|
| 1083 |
+
"num_tokens": 57318196.0,
|
| 1084 |
+
"reward": 0.24701710138469934,
|
| 1085 |
+
"reward_std": 0.3370856884866953,
|
| 1086 |
+
"rewards/cosine_scaled_reward": 0.2470170920714736,
|
| 1087 |
+
"rewards/format_reward": 0.0,
|
| 1088 |
+
"step": 77
|
| 1089 |
+
},
|
| 1090 |
+
{
|
| 1091 |
+
"clip_ratio": 0.0,
|
| 1092 |
+
"completion_length": 684.2477912902832,
|
| 1093 |
+
"epoch": 0.7908745247148289,
|
| 1094 |
+
"grad_norm": 0.14715959131717682,
|
| 1095 |
+
"learning_rate": 7.551350165304499e-07,
|
| 1096 |
+
"loss": 0.0548,
|
| 1097 |
+
"num_tokens": 58049058.0,
|
| 1098 |
+
"reward": 0.30690577276982367,
|
| 1099 |
+
"reward_std": 0.41454331763088703,
|
| 1100 |
+
"rewards/cosine_scaled_reward": 0.30690575792687014,
|
| 1101 |
+
"rewards/format_reward": 0.0,
|
| 1102 |
+
"step": 78
|
| 1103 |
+
},
|
| 1104 |
+
{
|
| 1105 |
+
"clip_ratio": 0.0,
|
| 1106 |
+
"completion_length": 651.327033996582,
|
| 1107 |
+
"epoch": 0.8010139416983524,
|
| 1108 |
+
"grad_norm": 0.13967670500278473,
|
| 1109 |
+
"learning_rate": 7.474191703716338e-07,
|
| 1110 |
+
"loss": 0.0039,
|
| 1111 |
+
"num_tokens": 58763207.0,
|
| 1112 |
+
"reward": 0.24645255785435438,
|
| 1113 |
+
"reward_std": 0.36919011175632477,
|
| 1114 |
+
"rewards/cosine_scaled_reward": 0.2464525457471609,
|
| 1115 |
+
"rewards/format_reward": 0.0,
|
| 1116 |
+
"step": 79
|
| 1117 |
+
},
|
| 1118 |
+
{
|
| 1119 |
+
"clip_ratio": 0.0,
|
| 1120 |
+
"completion_length": 620.6060562133789,
|
| 1121 |
+
"epoch": 0.8111533586818758,
|
| 1122 |
+
"grad_norm": 0.1529722511768341,
|
| 1123 |
+
"learning_rate": 7.396244933600284e-07,
|
| 1124 |
+
"loss": 0.0101,
|
| 1125 |
+
"num_tokens": 59443558.0,
|
| 1126 |
+
"reward": 0.25752069521695375,
|
| 1127 |
+
"reward_std": 0.4100553132593632,
|
| 1128 |
+
"rewards/cosine_scaled_reward": 0.25752068776637316,
|
| 1129 |
+
"rewards/format_reward": 0.0,
|
| 1130 |
+
"step": 80
|
| 1131 |
+
},
|
| 1132 |
+
{
|
| 1133 |
+
"clip_ratio": 0.0,
|
| 1134 |
+
"completion_length": 641.4230117797852,
|
| 1135 |
+
"epoch": 0.8212927756653993,
|
| 1136 |
+
"grad_norm": 0.1544528305530548,
|
| 1137 |
+
"learning_rate": 7.317534689775527e-07,
|
| 1138 |
+
"loss": 0.027,
|
| 1139 |
+
"num_tokens": 60137977.0,
|
| 1140 |
+
"reward": 0.22499833535403013,
|
| 1141 |
+
"reward_std": 0.406455984339118,
|
| 1142 |
+
"rewards/cosine_scaled_reward": 0.22499832697212696,
|
| 1143 |
+
"rewards/format_reward": 0.0,
|
| 1144 |
+
"step": 81
|
| 1145 |
+
},
|
| 1146 |
+
{
|
| 1147 |
+
"clip_ratio": 0.0,
|
| 1148 |
+
"completion_length": 616.1852912902832,
|
| 1149 |
+
"epoch": 0.8314321926489227,
|
| 1150 |
+
"grad_norm": 0.20284396409988403,
|
| 1151 |
+
"learning_rate": 7.238086050313562e-07,
|
| 1152 |
+
"loss": -0.0023,
|
| 1153 |
+
"num_tokens": 60825335.0,
|
| 1154 |
+
"reward": 0.1846146322786808,
|
| 1155 |
+
"reward_std": 0.42002810910344124,
|
| 1156 |
+
"rewards/cosine_scaled_reward": 0.1846146178431809,
|
| 1157 |
+
"rewards/format_reward": 0.0,
|
| 1158 |
+
"step": 82
|
| 1159 |
+
},
|
| 1160 |
+
{
|
| 1161 |
+
"clip_ratio": 0.0,
|
| 1162 |
+
"completion_length": 635.2466812133789,
|
| 1163 |
+
"epoch": 0.8415716096324461,
|
| 1164 |
+
"grad_norm": 0.163020059466362,
|
| 1165 |
+
"learning_rate": 7.157924328548002e-07,
|
| 1166 |
+
"loss": 0.0196,
|
| 1167 |
+
"num_tokens": 61528108.0,
|
| 1168 |
+
"reward": 0.2309768390841782,
|
| 1169 |
+
"reward_std": 0.4035301711410284,
|
| 1170 |
+
"rewards/cosine_scaled_reward": 0.2309768325649202,
|
| 1171 |
+
"rewards/format_reward": 0.0,
|
| 1172 |
+
"step": 83
|
| 1173 |
+
},
|
| 1174 |
+
{
|
| 1175 |
+
"clip_ratio": 0.0,
|
| 1176 |
+
"completion_length": 632.155158996582,
|
| 1177 |
+
"epoch": 0.8517110266159695,
|
| 1178 |
+
"grad_norm": 0.15208573639392853,
|
| 1179 |
+
"learning_rate": 7.077075065009433e-07,
|
| 1180 |
+
"loss": 0.0356,
|
| 1181 |
+
"num_tokens": 62227831.0,
|
| 1182 |
+
"reward": 0.29583334969356656,
|
| 1183 |
+
"reward_std": 0.4564223028719425,
|
| 1184 |
+
"rewards/cosine_scaled_reward": 0.29583333851769567,
|
| 1185 |
+
"rewards/format_reward": 0.0,
|
| 1186 |
+
"step": 84
|
| 1187 |
+
},
|
| 1188 |
+
{
|
| 1189 |
+
"clip_ratio": 0.0,
|
| 1190 |
+
"completion_length": 626.7265930175781,
|
| 1191 |
+
"epoch": 0.861850443599493,
|
| 1192 |
+
"grad_norm": 0.15748122334480286,
|
| 1193 |
+
"learning_rate": 6.995564019287869e-07,
|
| 1194 |
+
"loss": 0.0,
|
| 1195 |
+
"num_tokens": 62919194.0,
|
| 1196 |
+
"reward": 0.16931751649826765,
|
| 1197 |
+
"reward_std": 0.3771196622401476,
|
| 1198 |
+
"rewards/cosine_scaled_reward": 0.1693175071850419,
|
| 1199 |
+
"rewards/format_reward": 0.0,
|
| 1200 |
+
"step": 85
|
| 1201 |
+
},
|
| 1202 |
+
{
|
| 1203 |
+
"clip_ratio": 0.0,
|
| 1204 |
+
"completion_length": 665.608283996582,
|
| 1205 |
+
"epoch": 0.8719898605830165,
|
| 1206 |
+
"grad_norm": 0.18250492215156555,
|
| 1207 |
+
"learning_rate": 6.913417161825449e-07,
|
| 1208 |
+
"loss": -0.0308,
|
| 1209 |
+
"num_tokens": 63650491.0,
|
| 1210 |
+
"reward": 0.20493324054405093,
|
| 1211 |
+
"reward_std": 0.3730998281389475,
|
| 1212 |
+
"rewards/cosine_scaled_reward": 0.2049332265742123,
|
| 1213 |
+
"rewards/format_reward": 0.0,
|
| 1214 |
+
"step": 86
|
| 1215 |
+
},
|
| 1216 |
+
{
|
| 1217 |
+
"clip_ratio": 0.0,
|
| 1218 |
+
"completion_length": 622.6272583007812,
|
| 1219 |
+
"epoch": 0.8821292775665399,
|
| 1220 |
+
"grad_norm": 0.15512950718402863,
|
| 1221 |
+
"learning_rate": 6.830660665641897e-07,
|
| 1222 |
+
"loss": 0.0071,
|
| 1223 |
+
"num_tokens": 64338085.0,
|
| 1224 |
+
"reward": 0.2438387837028131,
|
| 1225 |
+
"reward_std": 0.40685577876865864,
|
| 1226 |
+
"rewards/cosine_scaled_reward": 0.24383877718355507,
|
| 1227 |
+
"rewards/format_reward": 0.0,
|
| 1228 |
+
"step": 87
|
| 1229 |
+
},
|
| 1230 |
+
{
|
| 1231 |
+
"clip_ratio": 0.0,
|
| 1232 |
+
"completion_length": 664.6975784301758,
|
| 1233 |
+
"epoch": 0.8922686945500634,
|
| 1234 |
+
"grad_norm": 0.9565799832344055,
|
| 1235 |
+
"learning_rate": 6.747320897995492e-07,
|
| 1236 |
+
"loss": 0.0461,
|
| 1237 |
+
"num_tokens": 65062646.0,
|
| 1238 |
+
"reward": 0.25703565403819084,
|
| 1239 |
+
"reward_std": 0.4041633568704128,
|
| 1240 |
+
"rewards/cosine_scaled_reward": 0.2570356400683522,
|
| 1241 |
+
"rewards/format_reward": 0.0,
|
| 1242 |
+
"step": 88
|
| 1243 |
+
},
|
| 1244 |
+
{
|
| 1245 |
+
"clip_ratio": 0.0,
|
| 1246 |
+
"completion_length": 588.4866333007812,
|
| 1247 |
+
"epoch": 0.9024081115335868,
|
| 1248 |
+
"grad_norm": 0.15928997099399567,
|
| 1249 |
+
"learning_rate": 6.66342441198212e-07,
|
| 1250 |
+
"loss": 0.003,
|
| 1251 |
+
"num_tokens": 65713834.0,
|
| 1252 |
+
"reward": 0.22450890811160207,
|
| 1253 |
+
"reward_std": 0.3792395293712616,
|
| 1254 |
+
"rewards/cosine_scaled_reward": 0.22450890205800533,
|
| 1255 |
+
"rewards/format_reward": 0.0,
|
| 1256 |
+
"step": 89
|
| 1257 |
+
},
|
| 1258 |
+
{
|
| 1259 |
+
"clip_ratio": 0.0,
|
| 1260 |
+
"completion_length": 617.6417694091797,
|
| 1261 |
+
"epoch": 0.9125475285171103,
|
| 1262 |
+
"grad_norm": 0.16051417589187622,
|
| 1263 |
+
"learning_rate": 6.578997938075125e-07,
|
| 1264 |
+
"loss": 0.0237,
|
| 1265 |
+
"num_tokens": 66391713.0,
|
| 1266 |
+
"reward": 0.17285537254065275,
|
| 1267 |
+
"reward_std": 0.4654933251440525,
|
| 1268 |
+
"rewards/cosine_scaled_reward": 0.17285536974668503,
|
| 1269 |
+
"rewards/format_reward": 0.0,
|
| 1270 |
+
"step": 90
|
| 1271 |
+
},
|
| 1272 |
+
{
|
| 1273 |
+
"clip_ratio": 0.0,
|
| 1274 |
+
"completion_length": 612.7511444091797,
|
| 1275 |
+
"epoch": 0.9226869455006337,
|
| 1276 |
+
"grad_norm": 0.1583762913942337,
|
| 1277 |
+
"learning_rate": 6.494068375608646e-07,
|
| 1278 |
+
"loss": -0.022,
|
| 1279 |
+
"num_tokens": 67075362.0,
|
| 1280 |
+
"reward": 0.2817280013114214,
|
| 1281 |
+
"reward_std": 0.3609468787908554,
|
| 1282 |
+
"rewards/cosine_scaled_reward": 0.28172798082232475,
|
| 1283 |
+
"rewards/format_reward": 0.0,
|
| 1284 |
+
"step": 91
|
| 1285 |
+
},
|
| 1286 |
+
{
|
| 1287 |
+
"clip_ratio": 0.0,
|
| 1288 |
+
"completion_length": 647.9185485839844,
|
| 1289 |
+
"epoch": 0.9328263624841572,
|
| 1290 |
+
"grad_norm": 0.16609789431095123,
|
| 1291 |
+
"learning_rate": 6.408662784207149e-07,
|
| 1292 |
+
"loss": 0.0336,
|
| 1293 |
+
"num_tokens": 67795273.0,
|
| 1294 |
+
"reward": 0.2746627312153578,
|
| 1295 |
+
"reward_std": 0.41118355095386505,
|
| 1296 |
+
"rewards/cosine_scaled_reward": 0.2746627163141966,
|
| 1297 |
+
"rewards/format_reward": 0.0,
|
| 1298 |
+
"step": 92
|
| 1299 |
+
},
|
| 1300 |
+
{
|
| 1301 |
+
"clip_ratio": 0.0,
|
| 1302 |
+
"completion_length": 586.2076110839844,
|
| 1303 |
+
"epoch": 0.9429657794676806,
|
| 1304 |
+
"grad_norm": 0.17962822318077087,
|
| 1305 |
+
"learning_rate": 6.322808375163895e-07,
|
| 1306 |
+
"loss": 0.0304,
|
| 1307 |
+
"num_tokens": 68447395.0,
|
| 1308 |
+
"reward": 0.2662193002179265,
|
| 1309 |
+
"reward_std": 0.4135792311280966,
|
| 1310 |
+
"rewards/cosine_scaled_reward": 0.2662192853167653,
|
| 1311 |
+
"rewards/format_reward": 0.0,
|
| 1312 |
+
"step": 93
|
| 1313 |
+
},
|
| 1314 |
+
{
|
| 1315 |
+
"clip_ratio": 0.0,
|
| 1316 |
+
"completion_length": 628.2835159301758,
|
| 1317 |
+
"epoch": 0.9531051964512041,
|
| 1318 |
+
"grad_norm": 0.15532483160495758,
|
| 1319 |
+
"learning_rate": 6.236532502771077e-07,
|
| 1320 |
+
"loss": -0.0041,
|
| 1321 |
+
"num_tokens": 69143937.0,
|
| 1322 |
+
"reward": 0.22241216152906418,
|
| 1323 |
+
"reward_std": 0.41293996572494507,
|
| 1324 |
+
"rewards/cosine_scaled_reward": 0.22241215035319328,
|
| 1325 |
+
"rewards/format_reward": 0.0,
|
| 1326 |
+
"step": 94
|
| 1327 |
+
},
|
| 1328 |
+
{
|
| 1329 |
+
"clip_ratio": 0.0,
|
| 1330 |
+
"completion_length": 581.4933280944824,
|
| 1331 |
+
"epoch": 0.9632446134347274,
|
| 1332 |
+
"grad_norm": 0.15709497034549713,
|
| 1333 |
+
"learning_rate": 6.149862655604403e-07,
|
| 1334 |
+
"loss": 0.0438,
|
| 1335 |
+
"num_tokens": 69790643.0,
|
| 1336 |
+
"reward": 0.2721885642968118,
|
| 1337 |
+
"reward_std": 0.40717606246471405,
|
| 1338 |
+
"rewards/cosine_scaled_reward": 0.2721885568462312,
|
| 1339 |
+
"rewards/format_reward": 0.0,
|
| 1340 |
+
"step": 95
|
| 1341 |
+
},
|
| 1342 |
+
{
|
| 1343 |
+
"clip_ratio": 0.0,
|
| 1344 |
+
"completion_length": 614.9408721923828,
|
| 1345 |
+
"epoch": 0.973384030418251,
|
| 1346 |
+
"grad_norm": 0.16992299258708954,
|
| 1347 |
+
"learning_rate": 6.062826447764883e-07,
|
| 1348 |
+
"loss": 0.037,
|
| 1349 |
+
"num_tokens": 70464374.0,
|
| 1350 |
+
"reward": 0.2846832238137722,
|
| 1351 |
+
"reward_std": 0.3609850201755762,
|
| 1352 |
+
"rewards/cosine_scaled_reward": 0.2846832163631916,
|
| 1353 |
+
"rewards/format_reward": 0.0,
|
| 1354 |
+
"step": 96
|
| 1355 |
+
},
|
| 1356 |
+
{
|
| 1357 |
+
"clip_ratio": 0.0,
|
| 1358 |
+
"completion_length": 633.0982398986816,
|
| 1359 |
+
"epoch": 0.9835234474017744,
|
| 1360 |
+
"grad_norm": 0.14964738488197327,
|
| 1361 |
+
"learning_rate": 5.975451610080642e-07,
|
| 1362 |
+
"loss": 0.0147,
|
| 1363 |
+
"num_tokens": 71159438.0,
|
| 1364 |
+
"reward": 0.19606254808604717,
|
| 1365 |
+
"reward_std": 0.3826338779181242,
|
| 1366 |
+
"rewards/cosine_scaled_reward": 0.19606254203245044,
|
| 1367 |
+
"rewards/format_reward": 0.0,
|
| 1368 |
+
"step": 97
|
| 1369 |
+
},
|
| 1370 |
+
{
|
| 1371 |
+
"clip_ratio": 0.0,
|
| 1372 |
+
"completion_length": 603.615234375,
|
| 1373 |
+
"epoch": 0.9936628643852978,
|
| 1374 |
+
"grad_norm": 0.16081441938877106,
|
| 1375 |
+
"learning_rate": 5.887765981271517e-07,
|
| 1376 |
+
"loss": 0.0428,
|
| 1377 |
+
"num_tokens": 71852378.0,
|
| 1378 |
+
"reward": 0.2125524217262864,
|
| 1379 |
+
"reward_std": 0.3838059641420841,
|
| 1380 |
+
"rewards/cosine_scaled_reward": 0.21255241055041552,
|
| 1381 |
+
"rewards/format_reward": 0.0,
|
| 1382 |
+
"step": 98
|
| 1383 |
+
},
|
| 1384 |
+
{
|
| 1385 |
+
"clip_ratio": 0.0,
|
| 1386 |
+
"completion_length": 590.7076225280762,
|
| 1387 |
+
"epoch": 1.0101394169835234,
|
| 1388 |
+
"grad_norm": 0.1603391319513321,
|
| 1389 |
+
"learning_rate": 5.7997974990793e-07,
|
| 1390 |
+
"loss": 0.0057,
|
| 1391 |
+
"num_tokens": 72510644.0,
|
| 1392 |
+
"reward": 0.2637674935394898,
|
| 1393 |
+
"reward_std": 0.4375726915895939,
|
| 1394 |
+
"rewards/cosine_scaled_reward": 0.26376748620532453,
|
| 1395 |
+
"rewards/format_reward": 0.0,
|
| 1396 |
+
"step": 99
|
| 1397 |
+
},
|
| 1398 |
+
{
|
| 1399 |
+
"clip_ratio": 0.0,
|
| 1400 |
+
"completion_length": 611.1406593322754,
|
| 1401 |
+
"epoch": 1.020278833967047,
|
| 1402 |
+
"grad_norm": 0.16759544610977173,
|
| 1403 |
+
"learning_rate": 5.711574191366427e-07,
|
| 1404 |
+
"loss": 0.0233,
|
| 1405 |
+
"num_tokens": 73185602.0,
|
| 1406 |
+
"reward": 0.24167153984308243,
|
| 1407 |
+
"reward_std": 0.3558240346610546,
|
| 1408 |
+
"rewards/cosine_scaled_reward": 0.2416715300641954,
|
| 1409 |
+
"rewards/format_reward": 0.0,
|
| 1410 |
+
"step": 100
|
| 1411 |
+
},
|
| 1412 |
+
{
|
| 1413 |
+
"clip_ratio": 0.0,
|
| 1414 |
+
"completion_length": 630.9598426818848,
|
| 1415 |
+
"epoch": 1.0304182509505704,
|
| 1416 |
+
"grad_norm": 0.16307219862937927,
|
| 1417 |
+
"learning_rate": 5.623124167185929e-07,
|
| 1418 |
+
"loss": 0.0067,
|
| 1419 |
+
"num_tokens": 73873470.0,
|
| 1420 |
+
"reward": 0.26334861340001225,
|
| 1421 |
+
"reward_std": 0.3806659858673811,
|
| 1422 |
+
"rewards/cosine_scaled_reward": 0.26334861130453646,
|
| 1423 |
+
"rewards/format_reward": 0.0,
|
| 1424 |
+
"step": 101
|
| 1425 |
+
},
|
| 1426 |
+
{
|
| 1427 |
+
"clip_ratio": 0.0,
|
| 1428 |
+
"completion_length": 563.8035888671875,
|
| 1429 |
+
"epoch": 1.0405576679340938,
|
| 1430 |
+
"grad_norm": 0.18253548443317413,
|
| 1431 |
+
"learning_rate": 5.534475607825565e-07,
|
| 1432 |
+
"loss": 0.0047,
|
| 1433 |
+
"num_tokens": 74505926.0,
|
| 1434 |
+
"reward": 0.24372864328324795,
|
| 1435 |
+
"reward_std": 0.3952293395996094,
|
| 1436 |
+
"rewards/cosine_scaled_reward": 0.2437286302447319,
|
| 1437 |
+
"rewards/format_reward": 0.0,
|
| 1438 |
+
"step": 102
|
| 1439 |
+
},
|
| 1440 |
+
{
|
| 1441 |
+
"clip_ratio": 0.0,
|
| 1442 |
+
"completion_length": 634.3750267028809,
|
| 1443 |
+
"epoch": 1.0506970849176172,
|
| 1444 |
+
"grad_norm": 0.16632695496082306,
|
| 1445 |
+
"learning_rate": 5.445656757828879e-07,
|
| 1446 |
+
"loss": -0.0255,
|
| 1447 |
+
"num_tokens": 75205630.0,
|
| 1448 |
+
"reward": 0.07845631497912109,
|
| 1449 |
+
"reward_std": 0.38748307153582573,
|
| 1450 |
+
"rewards/cosine_scaled_reward": 0.07845630997326225,
|
| 1451 |
+
"rewards/format_reward": 0.0,
|
| 1452 |
+
"step": 103
|
| 1453 |
+
},
|
| 1454 |
+
{
|
| 1455 |
+
"clip_ratio": 0.0,
|
| 1456 |
+
"completion_length": 601.0971221923828,
|
| 1457 |
+
"epoch": 1.0608365019011408,
|
| 1458 |
+
"grad_norm": 0.14292435348033905,
|
| 1459 |
+
"learning_rate": 5.356695915996161e-07,
|
| 1460 |
+
"loss": -0.0234,
|
| 1461 |
+
"num_tokens": 75878573.0,
|
| 1462 |
+
"reward": 0.1749452566727996,
|
| 1463 |
+
"reward_std": 0.44058991968631744,
|
| 1464 |
+
"rewards/cosine_scaled_reward": 0.17494524829089642,
|
| 1465 |
+
"rewards/format_reward": 0.0,
|
| 1466 |
+
"step": 104
|
| 1467 |
+
},
|
| 1468 |
+
{
|
| 1469 |
+
"clip_ratio": 0.0,
|
| 1470 |
+
"completion_length": 558.7801628112793,
|
| 1471 |
+
"epoch": 1.0709759188846641,
|
| 1472 |
+
"grad_norm": 0.16877137124538422,
|
| 1473 |
+
"learning_rate": 5.267621426368075e-07,
|
| 1474 |
+
"loss": 0.0282,
|
| 1475 |
+
"num_tokens": 76498456.0,
|
| 1476 |
+
"reward": 0.29683027416467667,
|
| 1477 |
+
"reward_std": 0.43279460445046425,
|
| 1478 |
+
"rewards/cosine_scaled_reward": 0.2968302555382252,
|
| 1479 |
+
"rewards/format_reward": 0.0,
|
| 1480 |
+
"step": 105
|
| 1481 |
+
},
|
| 1482 |
+
{
|
| 1483 |
+
"clip_ratio": 0.0,
|
| 1484 |
+
"completion_length": 506.5335006713867,
|
| 1485 |
+
"epoch": 1.0811153358681875,
|
| 1486 |
+
"grad_norm": 0.17591001093387604,
|
| 1487 |
+
"learning_rate": 5.178461669194903e-07,
|
| 1488 |
+
"loss": 0.0483,
|
| 1489 |
+
"num_tokens": 77079518.0,
|
| 1490 |
+
"reward": 0.3861896749585867,
|
| 1491 |
+
"reward_std": 0.3774683568626642,
|
| 1492 |
+
"rewards/cosine_scaled_reward": 0.3861896488815546,
|
| 1493 |
+
"rewards/format_reward": 0.0,
|
| 1494 |
+
"step": 106
|
| 1495 |
+
},
|
| 1496 |
+
{
|
| 1497 |
+
"clip_ratio": 0.0,
|
| 1498 |
+
"completion_length": 534.6607398986816,
|
| 1499 |
+
"epoch": 1.091254752851711,
|
| 1500 |
+
"grad_norm": 0.15835636854171753,
|
| 1501 |
+
"learning_rate": 5.08924505189423e-07,
|
| 1502 |
+
"loss": -0.0117,
|
| 1503 |
+
"num_tokens": 77684534.0,
|
| 1504 |
+
"reward": 0.28510209172964096,
|
| 1505 |
+
"reward_std": 0.31983859837055206,
|
| 1506 |
+
"rewards/cosine_scaled_reward": 0.28510207310318947,
|
| 1507 |
+
"rewards/format_reward": 0.0,
|
| 1508 |
+
"step": 107
|
| 1509 |
+
},
|
| 1510 |
+
{
|
| 1511 |
+
"clip_ratio": 0.0,
|
| 1512 |
+
"completion_length": 594.6439895629883,
|
| 1513 |
+
"epoch": 1.1013941698352345,
|
| 1514 |
+
"grad_norm": 0.20408962666988373,
|
| 1515 |
+
"learning_rate": 5e-07,
|
| 1516 |
+
"loss": 0.0081,
|
| 1517 |
+
"num_tokens": 78341455.0,
|
| 1518 |
+
"reward": 0.2976014781743288,
|
| 1519 |
+
"reward_std": 0.3984498456120491,
|
| 1520 |
+
"rewards/cosine_scaled_reward": 0.29760145768523216,
|
| 1521 |
+
"rewards/format_reward": 0.0,
|
| 1522 |
+
"step": 108
|
| 1523 |
+
},
|
| 1524 |
+
{
|
| 1525 |
+
"clip_ratio": 0.0,
|
| 1526 |
+
"completion_length": 630.444221496582,
|
| 1527 |
+
"epoch": 1.111533586818758,
|
| 1528 |
+
"grad_norm": 0.1533069759607315,
|
| 1529 |
+
"learning_rate": 4.91075494810577e-07,
|
| 1530 |
+
"loss": 0.0265,
|
| 1531 |
+
"num_tokens": 79024325.0,
|
| 1532 |
+
"reward": 0.25680719036608934,
|
| 1533 |
+
"reward_std": 0.4040773529559374,
|
| 1534 |
+
"rewards/cosine_scaled_reward": 0.25680717546492815,
|
| 1535 |
+
"rewards/format_reward": 0.0,
|
| 1536 |
+
"step": 109
|
| 1537 |
+
},
|
| 1538 |
+
{
|
| 1539 |
+
"clip_ratio": 0.0,
|
| 1540 |
+
"completion_length": 637.928596496582,
|
| 1541 |
+
"epoch": 1.1216730038022813,
|
| 1542 |
+
"grad_norm": 0.16302607953548431,
|
| 1543 |
+
"learning_rate": 4.821538330805098e-07,
|
| 1544 |
+
"loss": -0.0033,
|
| 1545 |
+
"num_tokens": 79722429.0,
|
| 1546 |
+
"reward": 0.2886841148138046,
|
| 1547 |
+
"reward_std": 0.3645545728504658,
|
| 1548 |
+
"rewards/cosine_scaled_reward": 0.28868410736322403,
|
| 1549 |
+
"rewards/format_reward": 0.0,
|
| 1550 |
+
"step": 110
|
| 1551 |
+
},
|
| 1552 |
+
{
|
| 1553 |
+
"clip_ratio": 0.0,
|
| 1554 |
+
"completion_length": 614.0390968322754,
|
| 1555 |
+
"epoch": 1.131812420785805,
|
| 1556 |
+
"grad_norm": 0.15493924915790558,
|
| 1557 |
+
"learning_rate": 4.732378573631924e-07,
|
| 1558 |
+
"loss": 0.0113,
|
| 1559 |
+
"num_tokens": 80395336.0,
|
| 1560 |
+
"reward": 0.25373244076035917,
|
| 1561 |
+
"reward_std": 0.3782533872872591,
|
| 1562 |
+
"rewards/cosine_scaled_reward": 0.2537324334261939,
|
| 1563 |
+
"rewards/format_reward": 0.0,
|
| 1564 |
+
"step": 111
|
| 1565 |
+
},
|
| 1566 |
+
{
|
| 1567 |
+
"clip_ratio": 0.0,
|
| 1568 |
+
"completion_length": 615.8136520385742,
|
| 1569 |
+
"epoch": 1.1419518377693283,
|
| 1570 |
+
"grad_norm": 1.329131841659546,
|
| 1571 |
+
"learning_rate": 4.643304084003838e-07,
|
| 1572 |
+
"loss": 0.028,
|
| 1573 |
+
"num_tokens": 81074553.0,
|
| 1574 |
+
"reward": 0.19473237730562687,
|
| 1575 |
+
"reward_std": 0.4509877860546112,
|
| 1576 |
+
"rewards/cosine_scaled_reward": 0.19473236822523177,
|
| 1577 |
+
"rewards/format_reward": 0.0,
|
| 1578 |
+
"step": 112
|
| 1579 |
+
},
|
| 1580 |
+
{
|
| 1581 |
+
"clip_ratio": 0.0,
|
| 1582 |
+
"completion_length": 605.2098503112793,
|
| 1583 |
+
"epoch": 1.1520912547528517,
|
| 1584 |
+
"grad_norm": 0.16996121406555176,
|
| 1585 |
+
"learning_rate": 4.55434324217112e-07,
|
| 1586 |
+
"loss": 0.0026,
|
| 1587 |
+
"num_tokens": 81746061.0,
|
| 1588 |
+
"reward": 0.1302806935273111,
|
| 1589 |
+
"reward_std": 0.38874428533017635,
|
| 1590 |
+
"rewards/cosine_scaled_reward": 0.13028068700805306,
|
| 1591 |
+
"rewards/format_reward": 0.0,
|
| 1592 |
+
"step": 113
|
| 1593 |
+
},
|
| 1594 |
+
{
|
| 1595 |
+
"clip_ratio": 0.0,
|
| 1596 |
+
"completion_length": 596.4888648986816,
|
| 1597 |
+
"epoch": 1.162230671736375,
|
| 1598 |
+
"grad_norm": 0.1666853278875351,
|
| 1599 |
+
"learning_rate": 4.4655243921744367e-07,
|
| 1600 |
+
"loss": -0.0409,
|
| 1601 |
+
"num_tokens": 82405475.0,
|
| 1602 |
+
"reward": 0.21928726602345705,
|
| 1603 |
+
"reward_std": 0.3403788786381483,
|
| 1604 |
+
"rewards/cosine_scaled_reward": 0.21928725019097328,
|
| 1605 |
+
"rewards/format_reward": 0.0,
|
| 1606 |
+
"step": 114
|
| 1607 |
+
},
|
| 1608 |
+
{
|
| 1609 |
+
"clip_ratio": 0.0,
|
| 1610 |
+
"completion_length": 564.8895378112793,
|
| 1611 |
+
"epoch": 1.1723700887198987,
|
| 1612 |
+
"grad_norm": 0.16143201291561127,
|
| 1613 |
+
"learning_rate": 4.37687583281407e-07,
|
| 1614 |
+
"loss": -0.0382,
|
| 1615 |
+
"num_tokens": 83038832.0,
|
| 1616 |
+
"reward": 0.21677181296399795,
|
| 1617 |
+
"reward_std": 0.35355726443231106,
|
| 1618 |
+
"rewards/cosine_scaled_reward": 0.21677180923870765,
|
| 1619 |
+
"rewards/format_reward": 0.0,
|
| 1620 |
+
"step": 115
|
| 1621 |
+
},
|
| 1622 |
+
{
|
| 1623 |
+
"clip_ratio": 0.0,
|
| 1624 |
+
"completion_length": 553.4587287902832,
|
| 1625 |
+
"epoch": 1.182509505703422,
|
| 1626 |
+
"grad_norm": 0.16877496242523193,
|
| 1627 |
+
"learning_rate": 4.2884258086335745e-07,
|
| 1628 |
+
"loss": 0.0313,
|
| 1629 |
+
"num_tokens": 83673971.0,
|
| 1630 |
+
"reward": 0.2728145425207913,
|
| 1631 |
+
"reward_std": 0.42727693170309067,
|
| 1632 |
+
"rewards/cosine_scaled_reward": 0.2728145350702107,
|
| 1633 |
+
"rewards/format_reward": 0.0,
|
| 1634 |
+
"step": 116
|
| 1635 |
+
},
|
| 1636 |
+
{
|
| 1637 |
+
"clip_ratio": 0.0,
|
| 1638 |
+
"completion_length": 550.5703315734863,
|
| 1639 |
+
"epoch": 1.1926489226869454,
|
| 1640 |
+
"grad_norm": 0.17830805480480194,
|
| 1641 |
+
"learning_rate": 4.2002025009206987e-07,
|
| 1642 |
+
"loss": 0.0008,
|
| 1643 |
+
"num_tokens": 84299234.0,
|
| 1644 |
+
"reward": 0.1771111206180649,
|
| 1645 |
+
"reward_std": 0.42238056659698486,
|
| 1646 |
+
"rewards/cosine_scaled_reward": 0.17711111830431037,
|
| 1647 |
+
"rewards/format_reward": 0.0,
|
| 1648 |
+
"step": 117
|
| 1649 |
+
},
|
| 1650 |
+
{
|
| 1651 |
+
"clip_ratio": 0.0,
|
| 1652 |
+
"completion_length": 588.3515815734863,
|
| 1653 |
+
"epoch": 1.202788339670469,
|
| 1654 |
+
"grad_norm": 0.17095865309238434,
|
| 1655 |
+
"learning_rate": 4.1122340187284845e-07,
|
| 1656 |
+
"loss": 0.0134,
|
| 1657 |
+
"num_tokens": 84948685.0,
|
| 1658 |
+
"reward": 0.20526447892189026,
|
| 1659 |
+
"reward_std": 0.43867457658052444,
|
| 1660 |
+
"rewards/cosine_scaled_reward": 0.20526447147130966,
|
| 1661 |
+
"rewards/format_reward": 0.0,
|
| 1662 |
+
"step": 118
|
| 1663 |
+
},
|
| 1664 |
+
{
|
| 1665 |
+
"clip_ratio": 0.0,
|
| 1666 |
+
"completion_length": 536.7544975280762,
|
| 1667 |
+
"epoch": 1.2129277566539924,
|
| 1668 |
+
"grad_norm": 0.1749478578567505,
|
| 1669 |
+
"learning_rate": 4.0245483899193586e-07,
|
| 1670 |
+
"loss": 0.0174,
|
| 1671 |
+
"num_tokens": 85556993.0,
|
| 1672 |
+
"reward": 0.24879271956160665,
|
| 1673 |
+
"reward_std": 0.38236525282263756,
|
| 1674 |
+
"rewards/cosine_scaled_reward": 0.24879270186647773,
|
| 1675 |
+
"rewards/format_reward": 0.0,
|
| 1676 |
+
"step": 119
|
| 1677 |
+
},
|
| 1678 |
+
{
|
| 1679 |
+
"clip_ratio": 0.0,
|
| 1680 |
+
"completion_length": 558.5870742797852,
|
| 1681 |
+
"epoch": 1.2230671736375158,
|
| 1682 |
+
"grad_norm": 1.2731257677078247,
|
| 1683 |
+
"learning_rate": 3.937173552235116e-07,
|
| 1684 |
+
"loss": -0.0515,
|
| 1685 |
+
"num_tokens": 86190295.0,
|
| 1686 |
+
"reward": 0.1734993983991444,
|
| 1687 |
+
"reward_std": 0.3362387754023075,
|
| 1688 |
+
"rewards/cosine_scaled_reward": 0.1734993918798864,
|
| 1689 |
+
"rewards/format_reward": 0.0,
|
| 1690 |
+
"step": 120
|
| 1691 |
+
},
|
| 1692 |
+
{
|
| 1693 |
+
"clip_ratio": 0.0,
|
| 1694 |
+
"completion_length": 564.9698867797852,
|
| 1695 |
+
"epoch": 1.2332065906210392,
|
| 1696 |
+
"grad_norm": 0.20518043637275696,
|
| 1697 |
+
"learning_rate": 3.850137344395598e-07,
|
| 1698 |
+
"loss": 0.008,
|
| 1699 |
+
"num_tokens": 86819724.0,
|
| 1700 |
+
"reward": 0.2829995099455118,
|
| 1701 |
+
"reward_std": 0.33430546894669533,
|
| 1702 |
+
"rewards/cosine_scaled_reward": 0.2829994931817055,
|
| 1703 |
+
"rewards/format_reward": 0.0,
|
| 1704 |
+
"step": 121
|
| 1705 |
+
},
|
| 1706 |
+
{
|
| 1707 |
+
"clip_ratio": 0.0,
|
| 1708 |
+
"completion_length": 628.7623138427734,
|
| 1709 |
+
"epoch": 1.2433460076045628,
|
| 1710 |
+
"grad_norm": 0.1478431224822998,
|
| 1711 |
+
"learning_rate": 3.763467497228922e-07,
|
| 1712 |
+
"loss": -0.0359,
|
| 1713 |
+
"num_tokens": 87521127.0,
|
| 1714 |
+
"reward": 0.16334752831608057,
|
| 1715 |
+
"reward_std": 0.3575255237519741,
|
| 1716 |
+
"rewards/cosine_scaled_reward": 0.16334752179682255,
|
| 1717 |
+
"rewards/format_reward": 0.0,
|
| 1718 |
+
"step": 122
|
| 1719 |
+
},
|
| 1720 |
+
{
|
| 1721 |
+
"clip_ratio": 0.0,
|
| 1722 |
+
"completion_length": 576.7277030944824,
|
| 1723 |
+
"epoch": 1.2534854245880862,
|
| 1724 |
+
"grad_norm": 0.20034752786159515,
|
| 1725 |
+
"learning_rate": 3.677191624836106e-07,
|
| 1726 |
+
"loss": 0.0205,
|
| 1727 |
+
"num_tokens": 88159587.0,
|
| 1728 |
+
"reward": 0.32477190531790257,
|
| 1729 |
+
"reward_std": 0.3922874592244625,
|
| 1730 |
+
"rewards/cosine_scaled_reward": 0.3247718848288059,
|
| 1731 |
+
"rewards/format_reward": 0.0,
|
| 1732 |
+
"step": 123
|
| 1733 |
+
},
|
| 1734 |
+
{
|
| 1735 |
+
"clip_ratio": 0.0,
|
| 1736 |
+
"completion_length": 611.7667617797852,
|
| 1737 |
+
"epoch": 1.2636248415716096,
|
| 1738 |
+
"grad_norm": 0.1693291813135147,
|
| 1739 |
+
"learning_rate": 3.591337215792851e-07,
|
| 1740 |
+
"loss": 0.0109,
|
| 1741 |
+
"num_tokens": 88835898.0,
|
| 1742 |
+
"reward": 0.23217581922654063,
|
| 1743 |
+
"reward_std": 0.34129480831325054,
|
| 1744 |
+
"rewards/cosine_scaled_reward": 0.23217580071650445,
|
| 1745 |
+
"rewards/format_reward": 0.0,
|
| 1746 |
+
"step": 124
|
| 1747 |
+
},
|
| 1748 |
+
{
|
| 1749 |
+
"clip_ratio": 0.0,
|
| 1750 |
+
"completion_length": 557.9799385070801,
|
| 1751 |
+
"epoch": 1.2737642585551332,
|
| 1752 |
+
"grad_norm": 0.1676970273256302,
|
| 1753 |
+
"learning_rate": 3.505931624391355e-07,
|
| 1754 |
+
"loss": 0.0225,
|
| 1755 |
+
"num_tokens": 89454112.0,
|
| 1756 |
+
"reward": 0.20786370153655298,
|
| 1757 |
+
"reward_std": 0.4651025366038084,
|
| 1758 |
+
"rewards/cosine_scaled_reward": 0.20786369033157825,
|
| 1759 |
+
"rewards/format_reward": 0.0,
|
| 1760 |
+
"step": 125
|
| 1761 |
+
},
|
| 1762 |
+
{
|
| 1763 |
+
"clip_ratio": 0.0,
|
| 1764 |
+
"completion_length": 545.6931037902832,
|
| 1765 |
+
"epoch": 1.2839036755386566,
|
| 1766 |
+
"grad_norm": 0.1678163707256317,
|
| 1767 |
+
"learning_rate": 3.421002061924876e-07,
|
| 1768 |
+
"loss": 0.0151,
|
| 1769 |
+
"num_tokens": 90069917.0,
|
| 1770 |
+
"reward": 0.2275780588388443,
|
| 1771 |
+
"reward_std": 0.4175031315535307,
|
| 1772 |
+
"rewards/cosine_scaled_reward": 0.22757804207503796,
|
| 1773 |
+
"rewards/format_reward": 0.0,
|
| 1774 |
+
"step": 126
|
| 1775 |
+
},
|
| 1776 |
+
{
|
| 1777 |
+
"clip_ratio": 0.0,
|
| 1778 |
+
"completion_length": 540.5000152587891,
|
| 1779 |
+
"epoch": 1.29404309252218,
|
| 1780 |
+
"grad_norm": 0.17660397291183472,
|
| 1781 |
+
"learning_rate": 3.3365755880178807e-07,
|
| 1782 |
+
"loss": 0.0301,
|
| 1783 |
+
"num_tokens": 90675909.0,
|
| 1784 |
+
"reward": 0.2635859549045563,
|
| 1785 |
+
"reward_std": 0.41772597655653954,
|
| 1786 |
+
"rewards/cosine_scaled_reward": 0.2635859474539757,
|
| 1787 |
+
"rewards/format_reward": 0.0,
|
| 1788 |
+
"step": 127
|
| 1789 |
+
},
|
| 1790 |
+
{
|
| 1791 |
+
"clip_ratio": 0.0,
|
| 1792 |
+
"completion_length": 594.244441986084,
|
| 1793 |
+
"epoch": 1.3041825095057034,
|
| 1794 |
+
"grad_norm": 0.17012734711170197,
|
| 1795 |
+
"learning_rate": 3.2526791020045087e-07,
|
| 1796 |
+
"loss": 0.0215,
|
| 1797 |
+
"num_tokens": 91340952.0,
|
| 1798 |
+
"reward": 0.149813212454319,
|
| 1799 |
+
"reward_std": 0.4470374099910259,
|
| 1800 |
+
"rewards/cosine_scaled_reward": 0.14981320640072227,
|
| 1801 |
+
"rewards/format_reward": 0.0,
|
| 1802 |
+
"step": 128
|
| 1803 |
+
},
|
| 1804 |
+
{
|
| 1805 |
+
"clip_ratio": 0.0,
|
| 1806 |
+
"completion_length": 542.3348503112793,
|
| 1807 |
+
"epoch": 1.3143219264892267,
|
| 1808 |
+
"grad_norm": 0.174094557762146,
|
| 1809 |
+
"learning_rate": 3.169339334358104e-07,
|
| 1810 |
+
"loss": 0.0523,
|
| 1811 |
+
"num_tokens": 91960228.0,
|
| 1812 |
+
"reward": 0.26255749678239226,
|
| 1813 |
+
"reward_std": 0.32982037030160427,
|
| 1814 |
+
"rewards/cosine_scaled_reward": 0.26255749305710196,
|
| 1815 |
+
"rewards/format_reward": 0.0,
|
| 1816 |
+
"step": 129
|
| 1817 |
+
},
|
| 1818 |
+
{
|
| 1819 |
+
"clip_ratio": 0.0,
|
| 1820 |
+
"completion_length": 567.3292694091797,
|
| 1821 |
+
"epoch": 1.3244613434727504,
|
| 1822 |
+
"grad_norm": 0.16861200332641602,
|
| 1823 |
+
"learning_rate": 3.086582838174551e-07,
|
| 1824 |
+
"loss": -0.0835,
|
| 1825 |
+
"num_tokens": 92601043.0,
|
| 1826 |
+
"reward": 0.1493756880190631,
|
| 1827 |
+
"reward_std": 0.3699100259691477,
|
| 1828 |
+
"rewards/cosine_scaled_reward": 0.1493756801019117,
|
| 1829 |
+
"rewards/format_reward": 0.0,
|
| 1830 |
+
"step": 130
|
| 1831 |
+
},
|
| 1832 |
+
{
|
| 1833 |
+
"clip_ratio": 0.0,
|
| 1834 |
+
"completion_length": 578.1283683776855,
|
| 1835 |
+
"epoch": 1.3346007604562737,
|
| 1836 |
+
"grad_norm": 1.8984020948410034,
|
| 1837 |
+
"learning_rate": 3.004435980712129e-07,
|
| 1838 |
+
"loss": -0.0221,
|
| 1839 |
+
"num_tokens": 93242510.0,
|
| 1840 |
+
"reward": 0.23330406891182065,
|
| 1841 |
+
"reward_std": 0.4134122505784035,
|
| 1842 |
+
"rewards/cosine_scaled_reward": 0.23330405401065946,
|
| 1843 |
+
"rewards/format_reward": 0.0,
|
| 1844 |
+
"step": 131
|
| 1845 |
+
},
|
| 1846 |
+
{
|
| 1847 |
+
"clip_ratio": 0.0,
|
| 1848 |
+
"completion_length": 538.444221496582,
|
| 1849 |
+
"epoch": 1.3447401774397973,
|
| 1850 |
+
"grad_norm": 0.1705993413925171,
|
| 1851 |
+
"learning_rate": 2.922924934990568e-07,
|
| 1852 |
+
"loss": 0.0105,
|
| 1853 |
+
"num_tokens": 93850108.0,
|
| 1854 |
+
"reward": 0.18292724061757326,
|
| 1855 |
+
"reward_std": 0.4480227939784527,
|
| 1856 |
+
"rewards/cosine_scaled_reward": 0.18292723130434752,
|
| 1857 |
+
"rewards/format_reward": 0.0,
|
| 1858 |
+
"step": 132
|
| 1859 |
+
},
|
| 1860 |
+
{
|
| 1861 |
+
"clip_ratio": 0.0,
|
| 1862 |
+
"completion_length": 524.5457763671875,
|
| 1863 |
+
"epoch": 1.3548795944233207,
|
| 1864 |
+
"grad_norm": 0.20817777514457703,
|
| 1865 |
+
"learning_rate": 2.8420756714519954e-07,
|
| 1866 |
+
"loss": 0.0035,
|
| 1867 |
+
"num_tokens": 94464405.0,
|
| 1868 |
+
"reward": 0.2242459380067885,
|
| 1869 |
+
"reward_std": 0.3740137480199337,
|
| 1870 |
+
"rewards/cosine_scaled_reward": 0.2242459305562079,
|
| 1871 |
+
"rewards/format_reward": 0.0,
|
| 1872 |
+
"step": 133
|
| 1873 |
+
},
|
| 1874 |
+
{
|
| 1875 |
+
"clip_ratio": 0.0,
|
| 1876 |
+
"completion_length": 542.7221260070801,
|
| 1877 |
+
"epoch": 1.3650190114068441,
|
| 1878 |
+
"grad_norm": 0.1813177764415741,
|
| 1879 |
+
"learning_rate": 2.7619139496864376e-07,
|
| 1880 |
+
"loss": -0.029,
|
| 1881 |
+
"num_tokens": 95086668.0,
|
| 1882 |
+
"reward": 0.13100900989957154,
|
| 1883 |
+
"reward_std": 0.3776312284171581,
|
| 1884 |
+
"rewards/cosine_scaled_reward": 0.13100900826975703,
|
| 1885 |
+
"rewards/format_reward": 0.0,
|
| 1886 |
+
"step": 134
|
| 1887 |
+
},
|
| 1888 |
+
{
|
| 1889 |
+
"clip_ratio": 0.0,
|
| 1890 |
+
"completion_length": 500.7243461608887,
|
| 1891 |
+
"epoch": 1.3751584283903675,
|
| 1892 |
+
"grad_norm": 0.18803931772708893,
|
| 1893 |
+
"learning_rate": 2.6824653102244727e-07,
|
| 1894 |
+
"loss": 0.0016,
|
| 1895 |
+
"num_tokens": 95669389.0,
|
| 1896 |
+
"reward": 0.189885966014117,
|
| 1897 |
+
"reward_std": 0.45219165831804276,
|
| 1898 |
+
"rewards/cosine_scaled_reward": 0.18988595390692353,
|
| 1899 |
+
"rewards/format_reward": 0.0,
|
| 1900 |
+
"step": 135
|
| 1901 |
+
},
|
| 1902 |
+
{
|
| 1903 |
+
"clip_ratio": 0.0,
|
| 1904 |
+
"completion_length": 512.5524864196777,
|
| 1905 |
+
"epoch": 1.385297845373891,
|
| 1906 |
+
"grad_norm": 0.21896012127399445,
|
| 1907 |
+
"learning_rate": 2.603755066399718e-07,
|
| 1908 |
+
"loss": -0.0237,
|
| 1909 |
+
"num_tokens": 96258716.0,
|
| 1910 |
+
"reward": 0.17706315219402313,
|
| 1911 |
+
"reward_std": 0.4132867194712162,
|
| 1912 |
+
"rewards/cosine_scaled_reward": 0.1770631456747651,
|
| 1913 |
+
"rewards/format_reward": 0.0,
|
| 1914 |
+
"step": 136
|
| 1915 |
+
},
|
| 1916 |
+
{
|
| 1917 |
+
"clip_ratio": 0.0,
|
| 1918 |
+
"completion_length": 611.787971496582,
|
| 1919 |
+
"epoch": 1.3954372623574145,
|
| 1920 |
+
"grad_norm": 0.16569751501083374,
|
| 1921 |
+
"learning_rate": 2.5258082962836614e-07,
|
| 1922 |
+
"loss": -0.0733,
|
| 1923 |
+
"num_tokens": 96941438.0,
|
| 1924 |
+
"reward": 0.07232440868392587,
|
| 1925 |
+
"reward_std": 0.3806895762681961,
|
| 1926 |
+
"rewards/cosine_scaled_reward": 0.07232440612278879,
|
| 1927 |
+
"rewards/format_reward": 0.0,
|
| 1928 |
+
"step": 137
|
| 1929 |
+
},
|
| 1930 |
+
{
|
| 1931 |
+
"clip_ratio": 0.0,
|
| 1932 |
+
"completion_length": 532.1986846923828,
|
| 1933 |
+
"epoch": 1.4055766793409379,
|
| 1934 |
+
"grad_norm": 0.18526431918144226,
|
| 1935 |
+
"learning_rate": 2.4486498346955023e-07,
|
| 1936 |
+
"loss": 0.0031,
|
| 1937 |
+
"num_tokens": 97541120.0,
|
| 1938 |
+
"reward": 0.18689691158942878,
|
| 1939 |
+
"reward_std": 0.4354799725115299,
|
| 1940 |
+
"rewards/cosine_scaled_reward": 0.18689690087921917,
|
| 1941 |
+
"rewards/format_reward": 0.0,
|
| 1942 |
+
"step": 138
|
| 1943 |
+
},
|
| 1944 |
+
{
|
| 1945 |
+
"clip_ratio": 0.0,
|
| 1946 |
+
"completion_length": 538.1942138671875,
|
| 1947 |
+
"epoch": 1.4157160963244613,
|
| 1948 |
+
"grad_norm": 0.1791687160730362,
|
| 1949 |
+
"learning_rate": 2.372304265289436e-07,
|
| 1950 |
+
"loss": -0.0459,
|
| 1951 |
+
"num_tokens": 98147638.0,
|
| 1952 |
+
"reward": 0.13275248603895307,
|
| 1953 |
+
"reward_std": 0.44364722445607185,
|
| 1954 |
+
"rewards/cosine_scaled_reward": 0.1327524788212031,
|
| 1955 |
+
"rewards/format_reward": 0.0,
|
| 1956 |
+
"step": 139
|
| 1957 |
+
},
|
| 1958 |
+
{
|
| 1959 |
+
"clip_ratio": 0.0,
|
| 1960 |
+
"completion_length": 511.8605079650879,
|
| 1961 |
+
"epoch": 1.4258555133079849,
|
| 1962 |
+
"grad_norm": 0.1867491602897644,
|
| 1963 |
+
"learning_rate": 2.2967959127220137e-07,
|
| 1964 |
+
"loss": 0.0068,
|
| 1965 |
+
"num_tokens": 98728225.0,
|
| 1966 |
+
"reward": 0.3189204416703433,
|
| 1967 |
+
"reward_std": 0.3536707144230604,
|
| 1968 |
+
"rewards/cosine_scaled_reward": 0.3189204225782305,
|
| 1969 |
+
"rewards/format_reward": 0.0,
|
| 1970 |
+
"step": 140
|
| 1971 |
+
},
|
| 1972 |
+
{
|
| 1973 |
+
"clip_ratio": 0.0,
|
| 1974 |
+
"completion_length": 527.9910926818848,
|
| 1975 |
+
"epoch": 1.4359949302915083,
|
| 1976 |
+
"grad_norm": 0.18118509650230408,
|
| 1977 |
+
"learning_rate": 2.2221488349019902e-07,
|
| 1978 |
+
"loss": -0.0149,
|
| 1979 |
+
"num_tokens": 99349665.0,
|
| 1980 |
+
"reward": 0.136872966773808,
|
| 1981 |
+
"reward_std": 0.42309295386075974,
|
| 1982 |
+
"rewards/cosine_scaled_reward": 0.13687296118587255,
|
| 1983 |
+
"rewards/format_reward": 0.0,
|
| 1984 |
+
"step": 141
|
| 1985 |
+
},
|
| 1986 |
+
{
|
| 1987 |
+
"clip_ratio": 0.0,
|
| 1988 |
+
"completion_length": 508.416316986084,
|
| 1989 |
+
"epoch": 1.4461343472750317,
|
| 1990 |
+
"grad_norm": 0.17504200339317322,
|
| 1991 |
+
"learning_rate": 2.1483868153251788e-07,
|
| 1992 |
+
"loss": -0.0121,
|
| 1993 |
+
"num_tokens": 99936750.0,
|
| 1994 |
+
"reward": 0.24867849051952362,
|
| 1995 |
+
"reward_std": 0.4239983968436718,
|
| 1996 |
+
"rewards/cosine_scaled_reward": 0.2486784835346043,
|
| 1997 |
+
"rewards/format_reward": 0.0,
|
| 1998 |
+
"step": 142
|
| 1999 |
+
},
|
| 2000 |
+
{
|
| 2001 |
+
"clip_ratio": 0.0,
|
| 2002 |
+
"completion_length": 571.9408798217773,
|
| 2003 |
+
"epoch": 1.456273764258555,
|
| 2004 |
+
"grad_norm": 0.2036023736000061,
|
| 2005 |
+
"learning_rate": 2.0755333554967346e-07,
|
| 2006 |
+
"loss": -0.0038,
|
| 2007 |
+
"num_tokens": 100586409.0,
|
| 2008 |
+
"reward": 0.11940343677997589,
|
| 2009 |
+
"reward_std": 0.5181355699896812,
|
| 2010 |
+
"rewards/cosine_scaled_reward": 0.11940343119204044,
|
| 2011 |
+
"rewards/format_reward": 0.0,
|
| 2012 |
+
"step": 143
|
| 2013 |
+
},
|
| 2014 |
+
{
|
| 2015 |
+
"clip_ratio": 0.0,
|
| 2016 |
+
"completion_length": 599.2779235839844,
|
| 2017 |
+
"epoch": 1.4664131812420786,
|
| 2018 |
+
"grad_norm": 0.17052923142910004,
|
| 2019 |
+
"learning_rate": 2.0036116674432652e-07,
|
| 2020 |
+
"loss": -0.0245,
|
| 2021 |
+
"num_tokens": 101259810.0,
|
| 2022 |
+
"reward": 0.12587101384997368,
|
| 2023 |
+
"reward_std": 0.3964061960577965,
|
| 2024 |
+
"rewards/cosine_scaled_reward": 0.12587100733071566,
|
| 2025 |
+
"rewards/format_reward": 0.0,
|
| 2026 |
+
"step": 144
|
| 2027 |
+
},
|
| 2028 |
+
{
|
| 2029 |
+
"clip_ratio": 0.0,
|
| 2030 |
+
"completion_length": 473.37948989868164,
|
| 2031 |
+
"epoch": 1.476552598225602,
|
| 2032 |
+
"grad_norm": 0.1972968876361847,
|
| 2033 |
+
"learning_rate": 1.9326446663172035e-07,
|
| 2034 |
+
"loss": -0.0245,
|
| 2035 |
+
"num_tokens": 101815438.0,
|
| 2036 |
+
"reward": 0.2772935745306313,
|
| 2037 |
+
"reward_std": 0.44544481486082077,
|
| 2038 |
+
"rewards/cosine_scaled_reward": 0.2772935628890991,
|
| 2039 |
+
"rewards/format_reward": 0.0,
|
| 2040 |
+
"step": 145
|
| 2041 |
+
},
|
| 2042 |
+
{
|
| 2043 |
+
"clip_ratio": 0.0,
|
| 2044 |
+
"completion_length": 479.25560760498047,
|
| 2045 |
+
"epoch": 1.4866920152091254,
|
| 2046 |
+
"grad_norm": 0.236617311835289,
|
| 2047 |
+
"learning_rate": 1.8626549630957395e-07,
|
| 2048 |
+
"loss": 0.0277,
|
| 2049 |
+
"num_tokens": 102372995.0,
|
| 2050 |
+
"reward": 0.2525997795164585,
|
| 2051 |
+
"reward_std": 0.4284665212035179,
|
| 2052 |
+
"rewards/cosine_scaled_reward": 0.25259976275265217,
|
| 2053 |
+
"rewards/format_reward": 0.0,
|
| 2054 |
+
"step": 146
|
| 2055 |
+
},
|
| 2056 |
+
{
|
| 2057 |
+
"clip_ratio": 0.0,
|
| 2058 |
+
"completion_length": 550.899585723877,
|
| 2059 |
+
"epoch": 1.496831432192649,
|
| 2060 |
+
"grad_norm": 0.17915715277194977,
|
| 2061 |
+
"learning_rate": 1.7936648573766954e-07,
|
| 2062 |
+
"loss": -0.0269,
|
| 2063 |
+
"num_tokens": 103003313.0,
|
| 2064 |
+
"reward": 0.16996209789067507,
|
| 2065 |
+
"reward_std": 0.42602597922086716,
|
| 2066 |
+
"rewards/cosine_scaled_reward": 0.1699620874132961,
|
| 2067 |
+
"rewards/format_reward": 0.0,
|
| 2068 |
+
"step": 147
|
| 2069 |
+
},
|
| 2070 |
+
{
|
| 2071 |
+
"clip_ratio": 0.0,
|
| 2072 |
+
"completion_length": 564.1919822692871,
|
| 2073 |
+
"epoch": 1.5069708491761724,
|
| 2074 |
+
"grad_norm": 0.18789808452129364,
|
| 2075 |
+
"learning_rate": 1.725696330273575e-07,
|
| 2076 |
+
"loss": -0.0676,
|
| 2077 |
+
"num_tokens": 103639173.0,
|
| 2078 |
+
"reward": 0.16659816866740584,
|
| 2079 |
+
"reward_std": 0.36924581229686737,
|
| 2080 |
+
"rewards/cosine_scaled_reward": 0.16659815539605916,
|
| 2081 |
+
"rewards/format_reward": 0.0,
|
| 2082 |
+
"step": 148
|
| 2083 |
+
},
|
| 2084 |
+
{
|
| 2085 |
+
"clip_ratio": 0.0,
|
| 2086 |
+
"completion_length": 487.6774711608887,
|
| 2087 |
+
"epoch": 1.5171102661596958,
|
| 2088 |
+
"grad_norm": 0.20230001211166382,
|
| 2089 |
+
"learning_rate": 1.65877103741212e-07,
|
| 2090 |
+
"loss": 0.0106,
|
| 2091 |
+
"num_tokens": 104200860.0,
|
| 2092 |
+
"reward": 0.2995794331654906,
|
| 2093 |
+
"reward_std": 0.4441289193928242,
|
| 2094 |
+
"rewards/cosine_scaled_reward": 0.2995794131420553,
|
| 2095 |
+
"rewards/format_reward": 0.0,
|
| 2096 |
+
"step": 149
|
| 2097 |
+
},
|
| 2098 |
+
{
|
| 2099 |
+
"clip_ratio": 0.0,
|
| 2100 |
+
"completion_length": 515.7678756713867,
|
| 2101 |
+
"epoch": 1.5272496831432192,
|
| 2102 |
+
"grad_norm": 0.20191656053066254,
|
| 2103 |
+
"learning_rate": 1.592910302030544e-07,
|
| 2104 |
+
"loss": 0.0207,
|
| 2105 |
+
"num_tokens": 104782292.0,
|
| 2106 |
+
"reward": 0.17506056488491595,
|
| 2107 |
+
"reward_std": 0.4823239389806986,
|
| 2108 |
+
"rewards/cosine_scaled_reward": 0.17506056022830307,
|
| 2109 |
+
"rewards/format_reward": 0.0,
|
| 2110 |
+
"step": 150
|
| 2111 |
+
},
|
| 2112 |
+
{
|
| 2113 |
+
"clip_ratio": 0.0,
|
| 2114 |
+
"completion_length": 517.0513648986816,
|
| 2115 |
+
"epoch": 1.5373891001267426,
|
| 2116 |
+
"grad_norm": 0.18813706934452057,
|
| 2117 |
+
"learning_rate": 1.5281351081856976e-07,
|
| 2118 |
+
"loss": 0.0014,
|
| 2119 |
+
"num_tokens": 105376826.0,
|
| 2120 |
+
"reward": 0.19976339329605253,
|
| 2121 |
+
"reward_std": 0.43244756758213043,
|
| 2122 |
+
"rewards/cosine_scaled_reward": 0.19976338025753648,
|
| 2123 |
+
"rewards/format_reward": 0.0,
|
| 2124 |
+
"step": 151
|
| 2125 |
+
},
|
| 2126 |
+
{
|
| 2127 |
+
"clip_ratio": 0.0,
|
| 2128 |
+
"completion_length": 517.2968978881836,
|
| 2129 |
+
"epoch": 1.5475285171102662,
|
| 2130 |
+
"grad_norm": 0.18564638495445251,
|
| 2131 |
+
"learning_rate": 1.4644660940672627e-07,
|
| 2132 |
+
"loss": -0.0032,
|
| 2133 |
+
"num_tokens": 105965916.0,
|
| 2134 |
+
"reward": 0.29871877538971603,
|
| 2135 |
+
"reward_std": 0.39689321629703045,
|
| 2136 |
+
"rewards/cosine_scaled_reward": 0.29871875676326454,
|
| 2137 |
+
"rewards/format_reward": 0.0,
|
| 2138 |
+
"step": 152
|
| 2139 |
+
},
|
| 2140 |
+
{
|
| 2141 |
+
"clip_ratio": 0.0,
|
| 2142 |
+
"completion_length": 506.71319580078125,
|
| 2143 |
+
"epoch": 1.5576679340937896,
|
| 2144 |
+
"grad_norm": 0.17704474925994873,
|
| 2145 |
+
"learning_rate": 1.4019235454221856e-07,
|
| 2146 |
+
"loss": -0.0185,
|
| 2147 |
+
"num_tokens": 106554003.0,
|
| 2148 |
+
"reward": 0.21053828485310078,
|
| 2149 |
+
"reward_std": 0.3987581580877304,
|
| 2150 |
+
"rewards/cosine_scaled_reward": 0.21053827833384275,
|
| 2151 |
+
"rewards/format_reward": 0.0,
|
| 2152 |
+
"step": 153
|
| 2153 |
+
},
|
| 2154 |
+
{
|
| 2155 |
+
"clip_ratio": 0.0,
|
| 2156 |
+
"completion_length": 475.9431037902832,
|
| 2157 |
+
"epoch": 1.5678073510773132,
|
| 2158 |
+
"grad_norm": 0.19738495349884033,
|
| 2159 |
+
"learning_rate": 1.3405273890913737e-07,
|
| 2160 |
+
"loss": 0.0262,
|
| 2161 |
+
"num_tokens": 107102464.0,
|
| 2162 |
+
"reward": 0.21143945213407278,
|
| 2163 |
+
"reward_std": 0.42565931007266045,
|
| 2164 |
+
"rewards/cosine_scaled_reward": 0.21143944095820189,
|
| 2165 |
+
"rewards/format_reward": 0.0,
|
| 2166 |
+
"step": 154
|
| 2167 |
+
},
|
| 2168 |
+
{
|
| 2169 |
+
"clip_ratio": 0.0,
|
| 2170 |
+
"completion_length": 508.6618537902832,
|
| 2171 |
+
"epoch": 1.5779467680608366,
|
| 2172 |
+
"grad_norm": 0.328647643327713,
|
| 2173 |
+
"learning_rate": 1.280297186660752e-07,
|
| 2174 |
+
"loss": 0.0234,
|
| 2175 |
+
"num_tokens": 107676601.0,
|
| 2176 |
+
"reward": 0.25387828424572945,
|
| 2177 |
+
"reward_std": 0.4254852868616581,
|
| 2178 |
+
"rewards/cosine_scaled_reward": 0.25387826934456825,
|
| 2179 |
+
"rewards/format_reward": 0.0,
|
| 2180 |
+
"step": 155
|
| 2181 |
+
},
|
| 2182 |
+
{
|
| 2183 |
+
"clip_ratio": 0.0,
|
| 2184 |
+
"completion_length": 587.3850631713867,
|
| 2185 |
+
"epoch": 1.58808618504436,
|
| 2186 |
+
"grad_norm": 0.17507341504096985,
|
| 2187 |
+
"learning_rate": 1.2212521282287093e-07,
|
| 2188 |
+
"loss": -0.0302,
|
| 2189 |
+
"num_tokens": 108322826.0,
|
| 2190 |
+
"reward": 0.1246091895736754,
|
| 2191 |
+
"reward_std": 0.43387965485453606,
|
| 2192 |
+
"rewards/cosine_scaled_reward": 0.12460918258875608,
|
| 2193 |
+
"rewards/format_reward": 0.0,
|
| 2194 |
+
"step": 156
|
| 2195 |
+
},
|
| 2196 |
+
{
|
| 2197 |
+
"clip_ratio": 0.0,
|
| 2198 |
+
"completion_length": 570.8884086608887,
|
| 2199 |
+
"epoch": 1.5982256020278833,
|
| 2200 |
+
"grad_norm": 0.17769315838813782,
|
| 2201 |
+
"learning_rate": 1.1634110262918717e-07,
|
| 2202 |
+
"loss": -0.0209,
|
| 2203 |
+
"num_tokens": 108978350.0,
|
| 2204 |
+
"reward": 0.17509674560278654,
|
| 2205 |
+
"reward_std": 0.3465359937399626,
|
| 2206 |
+
"rewards/cosine_scaled_reward": 0.17509673908352852,
|
| 2207 |
+
"rewards/format_reward": 0.0,
|
| 2208 |
+
"step": 157
|
| 2209 |
+
},
|
| 2210 |
+
{
|
| 2211 |
+
"clip_ratio": 0.0,
|
| 2212 |
+
"completion_length": 546.3772621154785,
|
| 2213 |
+
"epoch": 1.6083650190114067,
|
| 2214 |
+
"grad_norm": 0.18788230419158936,
|
| 2215 |
+
"learning_rate": 1.1067923097512255e-07,
|
| 2216 |
+
"loss": -0.0358,
|
| 2217 |
+
"num_tokens": 109598616.0,
|
| 2218 |
+
"reward": 0.12754715513437986,
|
| 2219 |
+
"reward_std": 0.4496493525803089,
|
| 2220 |
+
"rewards/cosine_scaled_reward": 0.12754714622860774,
|
| 2221 |
+
"rewards/format_reward": 0.0,
|
| 2222 |
+
"step": 158
|
| 2223 |
+
},
|
| 2224 |
+
{
|
| 2225 |
+
"clip_ratio": 0.0,
|
| 2226 |
+
"completion_length": 543.7288246154785,
|
| 2227 |
+
"epoch": 1.6185044359949303,
|
| 2228 |
+
"grad_norm": 0.2574346661567688,
|
| 2229 |
+
"learning_rate": 1.0514140180404202e-07,
|
| 2230 |
+
"loss": -0.0512,
|
| 2231 |
+
"num_tokens": 110214853.0,
|
| 2232 |
+
"reward": 0.08189231436699629,
|
| 2233 |
+
"reward_std": 0.41547736898064613,
|
| 2234 |
+
"rewards/cosine_scaled_reward": 0.08189231110736728,
|
| 2235 |
+
"rewards/format_reward": 0.0,
|
| 2236 |
+
"step": 159
|
| 2237 |
+
},
|
| 2238 |
+
{
|
| 2239 |
+
"clip_ratio": 0.0,
|
| 2240 |
+
"completion_length": 607.2890892028809,
|
| 2241 |
+
"epoch": 1.6286438529784537,
|
| 2242 |
+
"grad_norm": 0.22311408817768097,
|
| 2243 |
+
"learning_rate": 9.972937953781984e-08,
|
| 2244 |
+
"loss": -0.0035,
|
| 2245 |
+
"num_tokens": 110905952.0,
|
| 2246 |
+
"reward": 0.07692541873257142,
|
| 2247 |
+
"reward_std": 0.4941936247050762,
|
| 2248 |
+
"rewards/cosine_scaled_reward": 0.07692541686992627,
|
| 2249 |
+
"rewards/format_reward": 0.0,
|
| 2250 |
+
"step": 160
|
| 2251 |
+
},
|
| 2252 |
+
{
|
| 2253 |
+
"clip_ratio": 0.0,
|
| 2254 |
+
"completion_length": 528.2031402587891,
|
| 2255 |
+
"epoch": 1.6387832699619773,
|
| 2256 |
+
"grad_norm": 1.5113683938980103,
|
| 2257 |
+
"learning_rate": 9.444488851467041e-08,
|
| 2258 |
+
"loss": -0.0166,
|
| 2259 |
+
"num_tokens": 111496894.0,
|
| 2260 |
+
"reward": 0.23110088612884283,
|
| 2261 |
+
"reward_std": 0.44028190523386,
|
| 2262 |
+
"rewards/cosine_scaled_reward": 0.23110087541863322,
|
| 2263 |
+
"rewards/format_reward": 0.0,
|
| 2264 |
+
"step": 161
|
| 2265 |
+
},
|
| 2266 |
+
{
|
| 2267 |
+
"clip_ratio": 0.0,
|
| 2268 |
+
"completion_length": 523.2132034301758,
|
| 2269 |
+
"epoch": 1.6489226869455007,
|
| 2270 |
+
"grad_norm": 0.18456611037254333,
|
| 2271 |
+
"learning_rate": 8.928961243975436e-08,
|
| 2272 |
+
"loss": -0.0148,
|
| 2273 |
+
"num_tokens": 112098549.0,
|
| 2274 |
+
"reward": 0.16438766405917704,
|
| 2275 |
+
"reward_std": 0.4540070556104183,
|
| 2276 |
+
"rewards/cosine_scaled_reward": 0.16713074478320777,
|
| 2277 |
+
"rewards/format_reward": 0.0,
|
| 2278 |
+
"step": 162
|
| 2279 |
+
},
|
| 2280 |
+
{
|
| 2281 |
+
"clip_ratio": 0.0,
|
| 2282 |
+
"completion_length": 490.20092010498047,
|
| 2283 |
+
"epoch": 1.659062103929024,
|
| 2284 |
+
"grad_norm": 0.20410823822021484,
|
| 2285 |
+
"learning_rate": 8.426519384872732e-08,
|
| 2286 |
+
"loss": -0.0209,
|
| 2287 |
+
"num_tokens": 112659905.0,
|
| 2288 |
+
"reward": 0.2585650213295594,
|
| 2289 |
+
"reward_std": 0.42336183600127697,
|
| 2290 |
+
"rewards/cosine_scaled_reward": 0.25856500840745866,
|
| 2291 |
+
"rewards/format_reward": 0.0,
|
| 2292 |
+
"step": 163
|
| 2293 |
+
},
|
| 2294 |
+
{
|
| 2295 |
+
"clip_ratio": 0.0,
|
| 2296 |
+
"completion_length": 515.3370704650879,
|
| 2297 |
+
"epoch": 1.6692015209125475,
|
| 2298 |
+
"grad_norm": 0.2762053906917572,
|
| 2299 |
+
"learning_rate": 7.937323358440934e-08,
|
| 2300 |
+
"loss": 0.036,
|
| 2301 |
+
"num_tokens": 113252679.0,
|
| 2302 |
+
"reward": 0.21294681075960398,
|
| 2303 |
+
"reward_std": 0.4499225113540888,
|
| 2304 |
+
"rewards/cosine_scaled_reward": 0.2129468061029911,
|
| 2305 |
+
"rewards/format_reward": 0.0,
|
| 2306 |
+
"step": 164
|
| 2307 |
+
},
|
| 2308 |
+
{
|
| 2309 |
+
"clip_ratio": 0.0,
|
| 2310 |
+
"completion_length": 517.8939971923828,
|
| 2311 |
+
"epoch": 1.6793409378960709,
|
| 2312 |
+
"grad_norm": 0.18798772990703583,
|
| 2313 |
+
"learning_rate": 7.461529028673463e-08,
|
| 2314 |
+
"loss": 0.0349,
|
| 2315 |
+
"num_tokens": 113849520.0,
|
| 2316 |
+
"reward": 0.2390075612347573,
|
| 2317 |
+
"reward_std": 0.4681055396795273,
|
| 2318 |
+
"rewards/cosine_scaled_reward": 0.23900754831265658,
|
| 2319 |
+
"rewards/format_reward": 0.0,
|
| 2320 |
+
"step": 165
|
| 2321 |
+
},
|
| 2322 |
+
{
|
| 2323 |
+
"clip_ratio": 0.0,
|
| 2324 |
+
"completion_length": 557.4988975524902,
|
| 2325 |
+
"epoch": 1.6894803548795945,
|
| 2326 |
+
"grad_norm": 0.19520263373851776,
|
| 2327 |
+
"learning_rate": 6.999287989614971e-08,
|
| 2328 |
+
"loss": 0.0068,
|
| 2329 |
+
"num_tokens": 114481479.0,
|
| 2330 |
+
"reward": 0.11839688033796847,
|
| 2331 |
+
"reward_std": 0.5270655192434788,
|
| 2332 |
+
"rewards/cosine_scaled_reward": 0.11839687242172658,
|
| 2333 |
+
"rewards/format_reward": 0.0,
|
| 2334 |
+
"step": 166
|
| 2335 |
+
},
|
| 2336 |
+
{
|
| 2337 |
+
"clip_ratio": 0.0,
|
| 2338 |
+
"completion_length": 484.58261489868164,
|
| 2339 |
+
"epoch": 1.6996197718631179,
|
| 2340 |
+
"grad_norm": 0.21452556550502777,
|
| 2341 |
+
"learning_rate": 6.550747517061656e-08,
|
| 2342 |
+
"loss": 0.0029,
|
| 2343 |
+
"num_tokens": 115042209.0,
|
| 2344 |
+
"reward": 0.24680990353226662,
|
| 2345 |
+
"reward_std": 0.4576365016400814,
|
| 2346 |
+
"rewards/cosine_scaled_reward": 0.2468099007382989,
|
| 2347 |
+
"rewards/format_reward": 0.0,
|
| 2348 |
+
"step": 167
|
| 2349 |
+
},
|
| 2350 |
+
{
|
| 2351 |
+
"clip_ratio": 0.0,
|
| 2352 |
+
"completion_length": 516.2667655944824,
|
| 2353 |
+
"epoch": 1.7097591888466415,
|
| 2354 |
+
"grad_norm": 0.18506427109241486,
|
| 2355 |
+
"learning_rate": 6.116050521637218e-08,
|
| 2356 |
+
"loss": 0.019,
|
| 2357 |
+
"num_tokens": 115633112.0,
|
| 2358 |
+
"reward": 0.22252054838463664,
|
| 2359 |
+
"reward_std": 0.4747694581747055,
|
| 2360 |
+
"rewards/cosine_scaled_reward": 0.22252054512500763,
|
| 2361 |
+
"rewards/format_reward": 0.0,
|
| 2362 |
+
"step": 168
|
| 2363 |
+
},
|
| 2364 |
+
{
|
| 2365 |
+
"clip_ratio": 0.0,
|
| 2366 |
+
"completion_length": 511.237735748291,
|
| 2367 |
+
"epoch": 1.7198986058301649,
|
| 2368 |
+
"grad_norm": 0.19289632141590118,
|
| 2369 |
+
"learning_rate": 5.6953355032598795e-08,
|
| 2370 |
+
"loss": -0.0568,
|
| 2371 |
+
"num_tokens": 116228253.0,
|
| 2372 |
+
"reward": 0.08733667735941708,
|
| 2373 |
+
"reward_std": 0.49206770583987236,
|
| 2374 |
+
"rewards/cosine_scaled_reward": 0.08733667538035661,
|
| 2375 |
+
"rewards/format_reward": 0.0,
|
| 2376 |
+
"step": 169
|
| 2377 |
+
},
|
| 2378 |
+
{
|
| 2379 |
+
"clip_ratio": 0.0,
|
| 2380 |
+
"completion_length": 520.1964530944824,
|
| 2381 |
+
"epoch": 1.7300380228136882,
|
| 2382 |
+
"grad_norm": 0.19966185092926025,
|
| 2383 |
+
"learning_rate": 5.288736507014435e-08,
|
| 2384 |
+
"loss": -0.0308,
|
| 2385 |
+
"num_tokens": 116818349.0,
|
| 2386 |
+
"reward": 0.20070420764386654,
|
| 2387 |
+
"reward_std": 0.45568813756108284,
|
| 2388 |
+
"rewards/cosine_scaled_reward": 0.20070419739931822,
|
| 2389 |
+
"rewards/format_reward": 0.0,
|
| 2390 |
+
"step": 170
|
| 2391 |
+
},
|
| 2392 |
+
{
|
| 2393 |
+
"clip_ratio": 0.0,
|
| 2394 |
+
"completion_length": 482.51565170288086,
|
| 2395 |
+
"epoch": 1.7401774397972116,
|
| 2396 |
+
"grad_norm": 0.2040020376443863,
|
| 2397 |
+
"learning_rate": 4.896383080443933e-08,
|
| 2398 |
+
"loss": -0.0247,
|
| 2399 |
+
"num_tokens": 117375435.0,
|
| 2400 |
+
"reward": 0.13767189331701957,
|
| 2401 |
+
"reward_std": 0.42815835028886795,
|
| 2402 |
+
"rewards/cosine_scaled_reward": 0.13767188772908412,
|
| 2403 |
+
"rewards/format_reward": 0.0,
|
| 2404 |
+
"step": 171
|
| 2405 |
+
},
|
| 2406 |
+
{
|
| 2407 |
+
"clip_ratio": 0.0,
|
| 2408 |
+
"completion_length": 504.1607437133789,
|
| 2409 |
+
"epoch": 1.750316856780735,
|
| 2410 |
+
"grad_norm": 0.20132046937942505,
|
| 2411 |
+
"learning_rate": 4.518400232274078e-08,
|
| 2412 |
+
"loss": -0.0345,
|
| 2413 |
+
"num_tokens": 117957667.0,
|
| 2414 |
+
"reward": 0.17123566241934896,
|
| 2415 |
+
"reward_std": 0.3756715804338455,
|
| 2416 |
+
"rewards/cosine_scaled_reward": 0.17370464373379946,
|
| 2417 |
+
"rewards/format_reward": 0.0,
|
| 2418 |
+
"step": 172
|
| 2419 |
+
},
|
| 2420 |
+
{
|
| 2421 |
+
"clip_ratio": 0.0,
|
| 2422 |
+
"completion_length": 460.175235748291,
|
| 2423 |
+
"epoch": 1.7604562737642584,
|
| 2424 |
+
"grad_norm": 0.2837740480899811,
|
| 2425 |
+
"learning_rate": 4.1549083925840165e-08,
|
| 2426 |
+
"loss": -0.0105,
|
| 2427 |
+
"num_tokens": 118487432.0,
|
| 2428 |
+
"reward": 0.2758436817675829,
|
| 2429 |
+
"reward_std": 0.4501863084733486,
|
| 2430 |
+
"rewards/cosine_scaled_reward": 0.2758436668664217,
|
| 2431 |
+
"rewards/format_reward": 0.0,
|
| 2432 |
+
"step": 173
|
| 2433 |
+
},
|
| 2434 |
+
{
|
| 2435 |
+
"clip_ratio": 0.0,
|
| 2436 |
+
"completion_length": 496.6093940734863,
|
| 2437 |
+
"epoch": 1.770595690747782,
|
| 2438 |
+
"grad_norm": 0.18853703141212463,
|
| 2439 |
+
"learning_rate": 3.806023374435663e-08,
|
| 2440 |
+
"loss": 0.0034,
|
| 2441 |
+
"num_tokens": 119060922.0,
|
| 2442 |
+
"reward": 0.19881045899819583,
|
| 2443 |
+
"reward_std": 0.4718875251710415,
|
| 2444 |
+
"rewards/cosine_scaled_reward": 0.19881044793874025,
|
| 2445 |
+
"rewards/format_reward": 0.0,
|
| 2446 |
+
"step": 174
|
| 2447 |
+
},
|
| 2448 |
+
{
|
| 2449 |
+
"clip_ratio": 0.0,
|
| 2450 |
+
"completion_length": 499.9899787902832,
|
| 2451 |
+
"epoch": 1.7807351077313056,
|
| 2452 |
+
"grad_norm": 0.18940913677215576,
|
| 2453 |
+
"learning_rate": 3.4718563369743213e-08,
|
| 2454 |
+
"loss": -0.0418,
|
| 2455 |
+
"num_tokens": 119642241.0,
|
| 2456 |
+
"reward": 0.1533535374328494,
|
| 2457 |
+
"reward_std": 0.45827095210552216,
|
| 2458 |
+
"rewards/cosine_scaled_reward": 0.1533535262569785,
|
| 2459 |
+
"rewards/format_reward": 0.0,
|
| 2460 |
+
"step": 175
|
| 2461 |
+
},
|
| 2462 |
+
{
|
| 2463 |
+
"clip_ratio": 0.0,
|
| 2464 |
+
"completion_length": 483.29467010498047,
|
| 2465 |
+
"epoch": 1.790874524714829,
|
| 2466 |
+
"grad_norm": 0.3907623291015625,
|
| 2467 |
+
"learning_rate": 3.15251375001192e-08,
|
| 2468 |
+
"loss": 0.0006,
|
| 2469 |
+
"num_tokens": 120215377.0,
|
| 2470 |
+
"reward": 0.30862145498394966,
|
| 2471 |
+
"reward_std": 0.44223641976714134,
|
| 2472 |
+
"rewards/cosine_scaled_reward": 0.308621431235224,
|
| 2473 |
+
"rewards/format_reward": 0.0,
|
| 2474 |
+
"step": 176
|
| 2475 |
+
},
|
| 2476 |
+
{
|
| 2477 |
+
"clip_ratio": 0.0,
|
| 2478 |
+
"completion_length": 515.3270263671875,
|
| 2479 |
+
"epoch": 1.8010139416983524,
|
| 2480 |
+
"grad_norm": 0.18325906991958618,
|
| 2481 |
+
"learning_rate": 2.8480973601043955e-08,
|
| 2482 |
+
"loss": -0.0398,
|
| 2483 |
+
"num_tokens": 120811918.0,
|
| 2484 |
+
"reward": 0.24171121884137392,
|
| 2485 |
+
"reward_std": 0.36342719942331314,
|
| 2486 |
+
"rewards/cosine_scaled_reward": 0.24171120021492243,
|
| 2487 |
+
"rewards/format_reward": 0.0,
|
| 2488 |
+
"step": 177
|
| 2489 |
+
},
|
| 2490 |
+
{
|
| 2491 |
+
"clip_ratio": 0.0,
|
| 2492 |
+
"completion_length": 542.7254676818848,
|
| 2493 |
+
"epoch": 1.8111533586818758,
|
| 2494 |
+
"grad_norm": 0.17345182597637177,
|
| 2495 |
+
"learning_rate": 2.558704158134023e-08,
|
| 2496 |
+
"loss": -0.0589,
|
| 2497 |
+
"num_tokens": 121438224.0,
|
| 2498 |
+
"reward": 0.10904384450986981,
|
| 2499 |
+
"reward_std": 0.4528394937515259,
|
| 2500 |
+
"rewards/cosine_scaled_reward": 0.10904384264722466,
|
| 2501 |
+
"rewards/format_reward": 0.0,
|
| 2502 |
+
"step": 178
|
| 2503 |
+
},
|
| 2504 |
+
{
|
| 2505 |
+
"clip_ratio": 0.0,
|
| 2506 |
+
"completion_length": 543.6852951049805,
|
| 2507 |
+
"epoch": 1.8212927756653992,
|
| 2508 |
+
"grad_norm": 0.1887708306312561,
|
| 2509 |
+
"learning_rate": 2.2844263484068093e-08,
|
| 2510 |
+
"loss": -0.0194,
|
| 2511 |
+
"num_tokens": 122052366.0,
|
| 2512 |
+
"reward": 0.1703888149932027,
|
| 2513 |
+
"reward_std": 0.4111927505582571,
|
| 2514 |
+
"rewards/cosine_scaled_reward": 0.17038880474865437,
|
| 2515 |
+
"rewards/format_reward": 0.0,
|
| 2516 |
+
"step": 179
|
| 2517 |
+
},
|
| 2518 |
+
{
|
| 2519 |
+
"clip_ratio": 0.0,
|
| 2520 |
+
"completion_length": 485.0948905944824,
|
| 2521 |
+
"epoch": 1.8314321926489225,
|
| 2522 |
+
"grad_norm": 0.20806105434894562,
|
| 2523 |
+
"learning_rate": 2.025351319275137e-08,
|
| 2524 |
+
"loss": -0.0092,
|
| 2525 |
+
"num_tokens": 122617979.0,
|
| 2526 |
+
"reward": 0.22041516425088048,
|
| 2527 |
+
"reward_std": 0.39602686278522015,
|
| 2528 |
+
"rewards/cosine_scaled_reward": 0.22041516285389662,
|
| 2529 |
+
"rewards/format_reward": 0.0,
|
| 2530 |
+
"step": 180
|
| 2531 |
+
},
|
| 2532 |
+
{
|
| 2533 |
+
"clip_ratio": 0.0,
|
| 2534 |
+
"completion_length": 521.0446624755859,
|
| 2535 |
+
"epoch": 1.8415716096324461,
|
| 2536 |
+
"grad_norm": 0.1908193826675415,
|
| 2537 |
+
"learning_rate": 1.781561615294652e-08,
|
| 2538 |
+
"loss": -0.0301,
|
| 2539 |
+
"num_tokens": 123219299.0,
|
| 2540 |
+
"reward": 0.1940143120009452,
|
| 2541 |
+
"reward_std": 0.4348856508731842,
|
| 2542 |
+
"rewards/cosine_scaled_reward": 0.19401429709978402,
|
| 2543 |
+
"rewards/format_reward": 0.0,
|
| 2544 |
+
"step": 181
|
| 2545 |
+
},
|
| 2546 |
+
{
|
| 2547 |
+
"clip_ratio": 0.0,
|
| 2548 |
+
"completion_length": 570.9743461608887,
|
| 2549 |
+
"epoch": 1.8517110266159695,
|
| 2550 |
+
"grad_norm": 0.17504295706748962,
|
| 2551 |
+
"learning_rate": 1.553134910924636e-08,
|
| 2552 |
+
"loss": -0.0429,
|
| 2553 |
+
"num_tokens": 123866996.0,
|
| 2554 |
+
"reward": 0.08305117121199146,
|
| 2555 |
+
"reward_std": 0.41529295966029167,
|
| 2556 |
+
"rewards/cosine_scaled_reward": 0.08305116754490882,
|
| 2557 |
+
"rewards/format_reward": 0.0,
|
| 2558 |
+
"step": 182
|
| 2559 |
+
},
|
| 2560 |
+
{
|
| 2561 |
+
"clip_ratio": 0.0,
|
| 2562 |
+
"completion_length": 481.1294822692871,
|
| 2563 |
+
"epoch": 1.8618504435994931,
|
| 2564 |
+
"grad_norm": 0.20204633474349976,
|
| 2565 |
+
"learning_rate": 1.340143985779829e-08,
|
| 2566 |
+
"loss": -0.0051,
|
| 2567 |
+
"num_tokens": 124424848.0,
|
| 2568 |
+
"reward": 0.2932664602994919,
|
| 2569 |
+
"reward_std": 0.4205982144922018,
|
| 2570 |
+
"rewards/cosine_scaled_reward": 0.2932664528489113,
|
| 2571 |
+
"rewards/format_reward": 0.0,
|
| 2572 |
+
"step": 183
|
| 2573 |
+
},
|
| 2574 |
+
{
|
| 2575 |
+
"clip_ratio": 0.0,
|
| 2576 |
+
"completion_length": 498.0591735839844,
|
| 2577 |
+
"epoch": 1.8719898605830165,
|
| 2578 |
+
"grad_norm": 0.18216805160045624,
|
| 2579 |
+
"learning_rate": 1.1426567014420297e-08,
|
| 2580 |
+
"loss": -0.0097,
|
| 2581 |
+
"num_tokens": 124997813.0,
|
| 2582 |
+
"reward": 0.27011361718177795,
|
| 2583 |
+
"reward_std": 0.42340338230133057,
|
| 2584 |
+
"rewards/cosine_scaled_reward": 0.27011359576135874,
|
| 2585 |
+
"rewards/format_reward": 0.0,
|
| 2586 |
+
"step": 184
|
| 2587 |
+
},
|
| 2588 |
+
{
|
| 2589 |
+
"clip_ratio": 0.0,
|
| 2590 |
+
"completion_length": 513.774585723877,
|
| 2591 |
+
"epoch": 1.88212927756654,
|
| 2592 |
+
"grad_norm": 0.1906953901052475,
|
| 2593 |
+
"learning_rate": 9.607359798384784e-09,
|
| 2594 |
+
"loss": -0.0212,
|
| 2595 |
+
"num_tokens": 125573635.0,
|
| 2596 |
+
"reward": 0.15609056316316128,
|
| 2597 |
+
"reward_std": 0.46621261164546013,
|
| 2598 |
+
"rewards/cosine_scaled_reward": 0.15609055384993553,
|
| 2599 |
+
"rewards/format_reward": 0.0,
|
| 2600 |
+
"step": 185
|
| 2601 |
+
},
|
| 2602 |
+
{
|
| 2603 |
+
"clip_ratio": 0.0,
|
| 2604 |
+
"completion_length": 511.1183280944824,
|
| 2605 |
+
"epoch": 1.8922686945500633,
|
| 2606 |
+
"grad_norm": 0.20339244604110718,
|
| 2607 |
+
"learning_rate": 7.944397831941951e-09,
|
| 2608 |
+
"loss": 0.0108,
|
| 2609 |
+
"num_tokens": 126178093.0,
|
| 2610 |
+
"reward": 0.12280734069645405,
|
| 2611 |
+
"reward_std": 0.4304603859782219,
|
| 2612 |
+
"rewards/cosine_scaled_reward": 0.12280733417719603,
|
| 2613 |
+
"rewards/format_reward": 0.0,
|
| 2614 |
+
"step": 186
|
| 2615 |
+
},
|
| 2616 |
+
{
|
| 2617 |
+
"clip_ratio": 0.0,
|
| 2618 |
+
"completion_length": 583.9364051818848,
|
| 2619 |
+
"epoch": 1.9024081115335867,
|
| 2620 |
+
"grad_norm": 0.18367899954319,
|
| 2621 |
+
"learning_rate": 6.438210955644452e-09,
|
| 2622 |
+
"loss": -0.0444,
|
| 2623 |
+
"num_tokens": 126843724.0,
|
| 2624 |
+
"reward": 0.019926082342863083,
|
| 2625 |
+
"reward_std": 0.4689461179077625,
|
| 2626 |
+
"rewards/cosine_scaled_reward": 0.019926077569834888,
|
| 2627 |
+
"rewards/format_reward": 0.0,
|
| 2628 |
+
"step": 187
|
| 2629 |
+
},
|
| 2630 |
+
{
|
| 2631 |
+
"clip_ratio": 0.0,
|
| 2632 |
+
"completion_length": 523.2076110839844,
|
| 2633 |
+
"epoch": 1.9125475285171103,
|
| 2634 |
+
"grad_norm": 0.17626607418060303,
|
| 2635 |
+
"learning_rate": 5.0892790595336575e-09,
|
| 2636 |
+
"loss": -0.0197,
|
| 2637 |
+
"num_tokens": 127449534.0,
|
| 2638 |
+
"reward": 0.11797729413956404,
|
| 2639 |
+
"reward_std": 0.497987262904644,
|
| 2640 |
+
"rewards/cosine_scaled_reward": 0.11797728785313666,
|
| 2641 |
+
"rewards/format_reward": 0.0,
|
| 2642 |
+
"step": 188
|
| 2643 |
+
},
|
| 2644 |
+
{
|
| 2645 |
+
"clip_ratio": 0.0,
|
| 2646 |
+
"completion_length": 525.0011367797852,
|
| 2647 |
+
"epoch": 1.9226869455006337,
|
| 2648 |
+
"grad_norm": 0.1973433792591095,
|
| 2649 |
+
"learning_rate": 3.898031930240797e-09,
|
| 2650 |
+
"loss": -0.029,
|
| 2651 |
+
"num_tokens": 128044935.0,
|
| 2652 |
+
"reward": 0.10833488963544369,
|
| 2653 |
+
"reward_std": 0.4491695165634155,
|
| 2654 |
+
"rewards/cosine_scaled_reward": 0.1083348891697824,
|
| 2655 |
+
"rewards/format_reward": 0.0,
|
| 2656 |
+
"step": 189
|
| 2657 |
+
},
|
| 2658 |
+
{
|
| 2659 |
+
"clip_ratio": 0.0,
|
| 2660 |
+
"completion_length": 548.4966850280762,
|
| 2661 |
+
"epoch": 1.9328263624841573,
|
| 2662 |
+
"grad_norm": 0.17003753781318665,
|
| 2663 |
+
"learning_rate": 2.8648491140513264e-09,
|
| 2664 |
+
"loss": 0.0062,
|
| 2665 |
+
"num_tokens": 128662836.0,
|
| 2666 |
+
"reward": 0.20282268431037664,
|
| 2667 |
+
"reward_std": 0.4673178493976593,
|
| 2668 |
+
"rewards/cosine_scaled_reward": 0.20282267965376377,
|
| 2669 |
+
"rewards/format_reward": 0.0,
|
| 2670 |
+
"step": 190
|
| 2671 |
+
},
|
| 2672 |
+
{
|
| 2673 |
+
"clip_ratio": 0.0,
|
| 2674 |
+
"completion_length": 440.6149711608887,
|
| 2675 |
+
"epoch": 1.9429657794676807,
|
| 2676 |
+
"grad_norm": 0.21727675199508667,
|
| 2677 |
+
"learning_rate": 1.9900597959770505e-09,
|
| 2678 |
+
"loss": 0.0655,
|
| 2679 |
+
"num_tokens": 129182755.0,
|
| 2680 |
+
"reward": 0.40362408570945263,
|
| 2681 |
+
"reward_std": 0.34423849545419216,
|
| 2682 |
+
"rewards/cosine_scaled_reward": 0.403624065220356,
|
| 2683 |
+
"rewards/format_reward": 0.0,
|
| 2684 |
+
"step": 191
|
| 2685 |
+
},
|
| 2686 |
+
{
|
| 2687 |
+
"clip_ratio": 0.0,
|
| 2688 |
+
"completion_length": 541.2031555175781,
|
| 2689 |
+
"epoch": 1.953105196451204,
|
| 2690 |
+
"grad_norm": 0.19199801981449127,
|
| 2691 |
+
"learning_rate": 1.2739426948732424e-09,
|
| 2692 |
+
"loss": -0.0442,
|
| 2693 |
+
"num_tokens": 129801641.0,
|
| 2694 |
+
"reward": 0.19715721579268575,
|
| 2695 |
+
"reward_std": 0.4420705772936344,
|
| 2696 |
+
"rewards/cosine_scaled_reward": 0.19715720368549228,
|
| 2697 |
+
"rewards/format_reward": 0.0,
|
| 2698 |
+
"step": 192
|
| 2699 |
+
},
|
| 2700 |
+
{
|
| 2701 |
+
"clip_ratio": 0.0,
|
| 2702 |
+
"completion_length": 526.9609565734863,
|
| 2703 |
+
"epoch": 1.9632446134347274,
|
| 2704 |
+
"grad_norm": 0.22452791035175323,
|
| 2705 |
+
"learning_rate": 7.16725974635568e-10,
|
| 2706 |
+
"loss": 0.0124,
|
| 2707 |
+
"num_tokens": 130399638.0,
|
| 2708 |
+
"reward": 0.20976402005180717,
|
| 2709 |
+
"reward_std": 0.4755300283432007,
|
| 2710 |
+
"rewards/cosine_scaled_reward": 0.20976400654762983,
|
| 2711 |
+
"rewards/format_reward": 0.0,
|
| 2712 |
+
"step": 193
|
| 2713 |
+
},
|
| 2714 |
+
{
|
| 2715 |
+
"clip_ratio": 0.0,
|
| 2716 |
+
"completion_length": 550.6752510070801,
|
| 2717 |
+
"epoch": 1.9733840304182508,
|
| 2718 |
+
"grad_norm": 0.18797193467617035,
|
| 2719 |
+
"learning_rate": 3.185871715041255e-10,
|
| 2720 |
+
"loss": -0.035,
|
| 2721 |
+
"num_tokens": 131025891.0,
|
| 2722 |
+
"reward": 0.1596639035269618,
|
| 2723 |
+
"reward_std": 0.4088666681200266,
|
| 2724 |
+
"rewards/cosine_scaled_reward": 0.15966389642562717,
|
| 2725 |
+
"rewards/format_reward": 0.0,
|
| 2726 |
+
"step": 194
|
| 2727 |
+
},
|
| 2728 |
+
{
|
| 2729 |
+
"clip_ratio": 0.0,
|
| 2730 |
+
"completion_length": 447.8102798461914,
|
| 2731 |
+
"epoch": 1.9835234474017744,
|
| 2732 |
+
"grad_norm": 0.19335201382637024,
|
| 2733 |
+
"learning_rate": 7.96531374983589e-11,
|
| 2734 |
+
"loss": -0.0162,
|
| 2735 |
+
"num_tokens": 131547609.0,
|
| 2736 |
+
"reward": 0.2565318220295012,
|
| 2737 |
+
"reward_std": 0.46955981478095055,
|
| 2738 |
+
"rewards/cosine_scaled_reward": 0.25653181644156575,
|
| 2739 |
+
"rewards/format_reward": 0.0,
|
| 2740 |
+
"step": 195
|
| 2741 |
+
},
|
| 2742 |
+
{
|
| 2743 |
+
"clip_ratio": 0.0,
|
| 2744 |
+
"completion_length": 516.494140625,
|
| 2745 |
+
"epoch": 1.9936628643852978,
|
| 2746 |
+
"grad_norm": 0.180581733584404,
|
| 2747 |
+
"learning_rate": 0.0,
|
| 2748 |
+
"loss": -0.0093,
|
| 2749 |
+
"num_tokens": 132131440.0,
|
| 2750 |
+
"reward": 0.08059131819754839,
|
| 2751 |
+
"reward_std": 0.5266188904643059,
|
| 2752 |
+
"rewards/cosine_scaled_reward": 0.08059131726622581,
|
| 2753 |
+
"rewards/format_reward": 0.0,
|
| 2754 |
+
"step": 196
|
| 2755 |
+
},
|
| 2756 |
+
{
|
| 2757 |
+
"epoch": 1.9936628643852978,
|
| 2758 |
+
"step": 196,
|
| 2759 |
+
"total_flos": 0.0,
|
| 2760 |
+
"train_loss": -0.019965038798853137,
|
| 2761 |
+
"train_runtime": 31425.8764,
|
| 2762 |
+
"train_samples_per_second": 0.703,
|
| 2763 |
+
"train_steps_per_second": 0.006
|
| 2764 |
+
}
|
| 2765 |
+
],
|
| 2766 |
+
"logging_steps": 1,
|
| 2767 |
+
"max_steps": 196,
|
| 2768 |
+
"num_input_tokens_seen": 0,
|
| 2769 |
+
"num_train_epochs": 2,
|
| 2770 |
+
"save_steps": 500,
|
| 2771 |
+
"stateful_callbacks": {
|
| 2772 |
+
"TrainerControl": {
|
| 2773 |
+
"args": {
|
| 2774 |
+
"should_epoch_stop": false,
|
| 2775 |
+
"should_evaluate": false,
|
| 2776 |
+
"should_log": false,
|
| 2777 |
+
"should_save": true,
|
| 2778 |
+
"should_training_stop": true
|
| 2779 |
+
},
|
| 2780 |
+
"attributes": {}
|
| 2781 |
+
}
|
| 2782 |
+
},
|
| 2783 |
+
"total_flos": 0.0,
|
| 2784 |
+
"train_batch_size": 16,
|
| 2785 |
+
"trial_name": null,
|
| 2786 |
+
"trial_params": null
|
| 2787 |
+
}
|