Hugging Face's logo

MatchaLwc
/
Qwen-2.5-7B-Simple-RL

Text Generation
Transformers
Safetensors
qwen2
Generated from Trainer
trl
grpo
conversational
text-generation-inference
Model card Files
xet
Community
MatchaLwc commited on
Commit
ab27334
·
verified ·
1 Parent(s): fe84a3f

Model save

Browse files
Files changed (9) hide show
  1. README.md +1 -1
  2. all_results.json +4 -4
  3. model-00001-of-00004.safetensors +1 -1
  4. model-00002-of-00004.safetensors +1 -1
  5. model-00003-of-00004.safetensors +1 -1
  6. model-00004-of-00004.safetensors +1 -1
  7. train_results.json +4 -4
  8. trainer_state.json +296 -120
  9. training_args.bin +1 -1
README.md CHANGED
@@ -26,7 +26,7 @@ print(output["generated_text"])
26
 
27
  ## Training procedure
28
 
29
- [<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/1105645918-bit/huggingface/runs/thgy7jr8)
30
 
31
 
32
  This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).
 
26
 
27
  ## Training procedure
28
 
29
+ [<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/1105645918-bit/huggingface/runs/5kszzipd)
30
 
31
 
32
  This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).
all_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
  "total_flos": 0.0,
3
- "train_loss": 0.04167769432613819,
4
- "train_runtime": 16138.2279,
5
  "train_samples": 7500,
6
- "train_samples_per_second": 0.465,
7
- "train_steps_per_second": 0.004
8
  }
 
1
  {
2
  "total_flos": 0.0,
3
+ "train_loss": 0.03906208295056708,
4
+ "train_runtime": 48868.7895,
5
  "train_samples": 7500,
6
+ "train_samples_per_second": 0.153,
7
+ "train_steps_per_second": 0.002
8
  }
model-00001-of-00004.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:2856b69d81256ba736419c8dd7d3888c341bb99a3e8164c01dd1f38e04b9cb09
3
  size 4877660776
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a314cbd99ea28c0baf1978854a5a2d9089a428e9df780361641dfa38d152575f
3
  size 4877660776
model-00002-of-00004.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:c6e5d3b0b9b412cc9a3350df68d0c468958931cade7304e1a0089f1f91c0f4c7
3
  size 4932751008
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b4893621368e783a1298d601bcbcc5c406d65fa8988d263f773638e3122e10bf
3
  size 4932751008
model-00003-of-00004.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:62ffa3854f68b29c3d10ad0712992b1903c2a52d3067d2165ff6a6076dab9a3d
3
  size 4330865200
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ec456b58e1c879061b6150420662b6f8a378f79bb858e185609394393deddf2e
3
  size 4330865200
model-00004-of-00004.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:08175dbe9a323334bb68afd9308c38cc741d2c5e4c6747285084d52c9a153a77
3
  size 1089994880
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d6e26270c279ad0e741d288b14605086e13c44ee23c751b2243fa508df7c2f5b
3
  size 1089994880
train_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
  "total_flos": 0.0,
3
- "train_loss": 0.04167769432613819,
4
- "train_runtime": 16138.2279,
5
  "train_samples": 7500,
6
- "train_samples_per_second": 0.465,
7
- "train_steps_per_second": 0.004
8
  }
 
1
  {
2
  "total_flos": 0.0,
3
+ "train_loss": 0.03906208295056708,
4
+ "train_runtime": 48868.7895,
5
  "train_samples": 7500,
6
+ "train_samples_per_second": 0.153,
7
+ "train_steps_per_second": 0.002
8
  }
trainer_state.json CHANGED
@@ -1,200 +1,376 @@
1
  {
2
  "best_metric": null,
3
  "best_model_checkpoint": null,
4
- "epoch": 0.9893390191897654,
5
  "eval_steps": 100,
6
- "global_step": 58,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
12
  "clip_ratio": 0.0,
13
- "completion_length": 594.6719055175781,
14
- "epoch": 0.017057569296375266,
15
- "grad_norm": 0.39196768403053284,
16
  "kl": 0.0,
17
- "learning_rate": 5e-07,
18
- "loss": -0.0015,
19
- "reward": 0.5937500223517418,
20
- "reward_std": 0.36258383840322495,
21
- "rewards/accuracy_reward": 0.5937500223517418,
22
  "rewards/format_reward": 0.0,
23
  "step": 1
24
  },
25
  {
26
  "clip_ratio": 0.0,
27
- "completion_length": 606.3256425857544,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
28
  "epoch": 0.08528784648187633,
29
- "grad_norm": 4.861485481262207,
30
- "kl": 0.00030231475830078125,
31
  "learning_rate": 2.5e-06,
32
- "loss": 0.0243,
33
- "reward": 0.5920759215950966,
34
- "reward_std": 0.3550507538020611,
35
- "rewards/accuracy_reward": 0.5920759215950966,
36
  "rewards/format_reward": 0.0,
37
- "step": 5
38
  },
39
  {
40
  "clip_ratio": 0.0,
41
- "completion_length": 615.3873008728027,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
42
  "epoch": 0.17057569296375266,
43
- "grad_norm": 1.1821469068527222,
44
- "kl": 0.005576515197753906,
45
- "learning_rate": 2.956412726139078e-06,
46
- "loss": 0.0514,
47
- "reward": 0.6691964574158191,
48
- "reward_std": 0.3115074638277292,
49
- "rewards/accuracy_reward": 0.6691964574158191,
50
  "rewards/format_reward": 0.0,
51
- "step": 10
52
  },
53
  {
54
  "clip_ratio": 0.0,
55
- "completion_length": 606.4236862182618,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
56
  "epoch": 0.255863539445629,
57
- "grad_norm": 0.18134506046772003,
58
- "kl": 0.010854721069335938,
59
- "learning_rate": 2.7836719084521715e-06,
60
- "loss": 0.0663,
61
- "reward": 0.7477678894996643,
62
- "reward_std": 0.24137643072754145,
63
- "rewards/accuracy_reward": 0.7477678894996643,
64
  "rewards/format_reward": 0.0,
65
- "step": 15
66
  },
67
  {
68
  "clip_ratio": 0.0,
69
- "completion_length": 597.0582832336426,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
70
  "epoch": 0.3411513859275053,
71
- "grad_norm": 0.1626371443271637,
72
- "kl": 0.019635581970214845,
73
- "learning_rate": 2.4946839873611927e-06,
74
- "loss": 0.0505,
75
- "reward": 0.7520089641213417,
76
- "reward_std": 0.22885524686425923,
77
- "rewards/accuracy_reward": 0.7520089641213417,
78
  "rewards/format_reward": 0.0,
79
- "step": 20
80
  },
81
  {
82
  "clip_ratio": 0.0,
83
- "completion_length": 598.8727935791015,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
84
  "epoch": 0.42643923240938164,
85
- "grad_norm": 0.19548186659812927,
86
- "kl": 0.026526451110839844,
87
- "learning_rate": 2.1156192081791355e-06,
88
- "loss": 0.0479,
89
- "reward": 0.7502232506871224,
90
- "reward_std": 0.2170567562803626,
91
- "rewards/accuracy_reward": 0.7502232506871224,
92
  "rewards/format_reward": 0.0,
93
- "step": 25
94
  },
95
  {
96
  "clip_ratio": 0.0,
97
- "completion_length": 600.4873069763183,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
98
  "epoch": 0.511727078891258,
99
- "grad_norm": 0.11407989263534546,
100
- "kl": 0.004177284240722656,
101
- "learning_rate": 1.6808050203829845e-06,
102
- "loss": 0.0307,
103
- "reward": 0.7473214626312256,
104
- "reward_std": 0.1896090098656714,
105
- "rewards/accuracy_reward": 0.7473214626312256,
106
  "rewards/format_reward": 0.0,
107
- "step": 30
 
 
 
 
 
 
 
 
 
 
 
 
 
 
108
  },
109
  {
110
  "clip_ratio": 0.0,
111
- "completion_length": 600.4290420532227,
112
  "epoch": 0.5970149253731343,
113
- "grad_norm": 0.10130893439054489,
114
- "kl": 0.004094696044921875,
115
- "learning_rate": 1.2296174432791415e-06,
116
- "loss": 0.0357,
117
- "reward": 0.7363839589059353,
118
- "reward_std": 0.1966801879927516,
119
- "rewards/accuracy_reward": 0.7363839589059353,
120
  "rewards/format_reward": 0.0,
121
- "step": 35
 
 
 
 
 
 
 
 
 
 
 
 
 
 
122
  },
123
  {
124
  "clip_ratio": 0.0,
125
- "completion_length": 587.7493537902832,
126
  "epoch": 0.6823027718550106,
127
- "grad_norm": 0.160549134016037,
128
- "kl": 0.005101394653320312,
129
- "learning_rate": 8.029152419343472e-07,
130
- "loss": 0.046,
131
- "reward": 0.7560268193483353,
132
- "reward_std": 0.20517763923853635,
133
- "rewards/accuracy_reward": 0.7560268193483353,
134
  "rewards/format_reward": 0.0,
135
- "step": 40
 
 
 
 
 
 
 
 
 
 
 
 
 
 
136
  },
137
  {
138
  "clip_ratio": 0.0,
139
- "completion_length": 600.7799362182617,
140
  "epoch": 0.767590618336887,
141
- "grad_norm": 0.11112036556005478,
142
- "kl": 0.009288406372070313,
143
- "learning_rate": 4.3933982822017883e-07,
144
- "loss": 0.0369,
145
- "reward": 0.7412946730852127,
146
- "reward_std": 0.19938444197177888,
147
- "rewards/accuracy_reward": 0.7412946730852127,
148
  "rewards/format_reward": 0.0,
149
- "step": 45
150
  },
151
  {
152
  "clip_ratio": 0.0,
153
- "completion_length": 600.3531517028808,
 
 
 
 
 
 
 
 
 
 
 
 
154
  "epoch": 0.8528784648187633,
155
- "grad_norm": 0.15258711576461792,
156
- "kl": 0.005424880981445312,
157
- "learning_rate": 1.718159615201853e-07,
158
- "loss": 0.041,
159
- "reward": 0.7310268193483352,
160
- "reward_std": 0.20577662400901317,
161
- "rewards/accuracy_reward": 0.7310268193483352,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
162
  "rewards/format_reward": 0.0,
163
- "step": 50
164
  },
165
  {
166
  "clip_ratio": 0.0,
167
- "completion_length": 599.8002517700195,
168
  "epoch": 0.9381663113006397,
169
- "grad_norm": 0.10871832072734833,
170
- "kl": 0.00552825927734375,
171
- "learning_rate": 2.4570139579284723e-08,
172
- "loss": 0.043,
173
- "reward": 0.7665178939700127,
174
- "reward_std": 0.2081836313009262,
175
- "rewards/accuracy_reward": 0.7665178939700127,
176
  "rewards/format_reward": 0.0,
177
- "step": 55
 
 
 
 
 
 
 
 
 
 
 
 
 
 
178
  },
179
  {
180
  "clip_ratio": 0.0,
181
- "completion_length": 585.7837626139323,
182
- "epoch": 0.9893390191897654,
183
- "kl": 0.0075321197509765625,
184
- "reward": 0.771205392976602,
185
- "reward_std": 0.18078403733670712,
186
- "rewards/accuracy_reward": 0.771205392976602,
187
  "rewards/format_reward": 0.0,
188
- "step": 58,
189
  "total_flos": 0.0,
190
- "train_loss": 0.04167769432613819,
191
- "train_runtime": 16138.2279,
192
- "train_samples_per_second": 0.465,
193
- "train_steps_per_second": 0.004
194
  }
195
  ],
196
  "logging_steps": 5,
197
- "max_steps": 58,
198
  "num_input_tokens_seen": 0,
199
  "num_train_epochs": 1,
200
  "save_steps": 500,
 
1
  {
2
  "best_metric": null,
3
  "best_model_checkpoint": null,
4
+ "epoch": 0.997867803837953,
5
  "eval_steps": 100,
6
+ "global_step": 117,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
10
  "log_history": [
11
  {
12
  "clip_ratio": 0.0,
13
+ "completion_length": 620.6149749755859,
14
+ "epoch": 0.008528784648187633,
15
+ "grad_norm": 0.43470078706741333,
16
  "kl": 0.0,
17
+ "learning_rate": 2.5e-07,
18
+ "loss": 0.0087,
19
+ "reward": 0.611607164144516,
20
+ "reward_std": 0.3698773570358753,
21
+ "rewards/accuracy_reward": 0.611607164144516,
22
  "rewards/format_reward": 0.0,
23
  "step": 1
24
  },
25
  {
26
  "clip_ratio": 0.0,
27
+ "completion_length": 593.875862121582,
28
+ "epoch": 0.042643923240938165,
29
+ "grad_norm": 0.43776455521583557,
30
+ "kl": 0.00015407800674438477,
31
+ "learning_rate": 1.25e-06,
32
+ "loss": 0.0231,
33
+ "reward": 0.6205357443541288,
34
+ "reward_std": 0.37901367666199803,
35
+ "rewards/accuracy_reward": 0.6205357443541288,
36
+ "rewards/format_reward": 0.0,
37
+ "step": 5
38
+ },
39
+ {
40
+ "clip_ratio": 0.0,
41
+ "completion_length": 616.8395332336426,
42
  "epoch": 0.08528784648187633,
43
+ "grad_norm": 0.8126201033592224,
44
+ "kl": 0.002016162872314453,
45
  "learning_rate": 2.5e-06,
46
+ "loss": 0.043,
47
+ "reward": 0.6250000275671482,
48
+ "reward_std": 0.347636329382658,
49
+ "rewards/accuracy_reward": 0.6250000275671482,
50
  "rewards/format_reward": 0.0,
51
+ "step": 10
52
  },
53
  {
54
  "clip_ratio": 0.0,
55
+ "completion_length": 610.7120796203614,
56
+ "epoch": 0.1279317697228145,
57
+ "grad_norm": 0.1547478586435318,
58
+ "kl": 0.013087844848632813,
59
+ "learning_rate": 2.993961440992859e-06,
60
+ "loss": 0.0896,
61
+ "reward": 0.7162946730852127,
62
+ "reward_std": 0.2822846982628107,
63
+ "rewards/accuracy_reward": 0.7162946730852127,
64
+ "rewards/format_reward": 0.0,
65
+ "step": 15
66
+ },
67
+ {
68
+ "clip_ratio": 0.0,
69
+ "completion_length": 612.700473022461,
70
  "epoch": 0.17057569296375266,
71
+ "grad_norm": 0.24769772589206696,
72
+ "kl": 0.01591835021972656,
73
+ "learning_rate": 2.957235057439301e-06,
74
+ "loss": 0.0769,
75
+ "reward": 0.7506696745753288,
76
+ "reward_std": 0.23058689776808022,
77
+ "rewards/accuracy_reward": 0.7506696745753288,
78
  "rewards/format_reward": 0.0,
79
+ "step": 20
80
  },
81
  {
82
  "clip_ratio": 0.0,
83
+ "completion_length": 575.4734634399414,
84
+ "epoch": 0.21321961620469082,
85
+ "grad_norm": 1.411234736442566,
86
+ "kl": 0.006292724609375,
87
+ "learning_rate": 2.887956450710995e-06,
88
+ "loss": 0.0598,
89
+ "reward": 0.8000000357627869,
90
+ "reward_std": 0.18773337844759225,
91
+ "rewards/accuracy_reward": 0.8000000357627869,
92
+ "rewards/format_reward": 0.0,
93
+ "step": 25
94
+ },
95
+ {
96
+ "clip_ratio": 0.0,
97
+ "completion_length": 579.8424324035644,
98
  "epoch": 0.255863539445629,
99
+ "grad_norm": 0.17543800175189972,
100
+ "kl": 0.004089736938476562,
101
+ "learning_rate": 2.7876731904027993e-06,
102
+ "loss": 0.0551,
103
+ "reward": 0.7698661036789417,
104
+ "reward_std": 0.20191936586052178,
105
+ "rewards/accuracy_reward": 0.7698661036789417,
106
  "rewards/format_reward": 0.0,
107
+ "step": 30
108
  },
109
  {
110
  "clip_ratio": 0.0,
111
+ "completion_length": 577.5357452392578,
112
+ "epoch": 0.29850746268656714,
113
+ "grad_norm": 0.14460965991020203,
114
+ "kl": 0.004575347900390625,
115
+ "learning_rate": 2.6586254388368995e-06,
116
+ "loss": 0.0494,
117
+ "reward": 0.7982143223285675,
118
+ "reward_std": 0.17998763117939234,
119
+ "rewards/accuracy_reward": 0.7982143223285675,
120
+ "rewards/format_reward": 0.0,
121
+ "step": 35
122
+ },
123
+ {
124
+ "clip_ratio": 0.0,
125
+ "completion_length": 573.0772575378418,
126
  "epoch": 0.3411513859275053,
127
+ "grad_norm": 0.10168638080358505,
128
+ "kl": 0.004691314697265625,
129
+ "learning_rate": 2.5036959095382875e-06,
130
+ "loss": 0.0403,
131
+ "reward": 0.7850446775555611,
132
+ "reward_std": 0.1875380737707019,
133
+ "rewards/accuracy_reward": 0.7850446775555611,
134
  "rewards/format_reward": 0.0,
135
+ "step": 40
136
  },
137
  {
138
  "clip_ratio": 0.0,
139
+ "completion_length": 583.6614151000977,
140
+ "epoch": 0.3837953091684435,
141
+ "grad_norm": 0.15747234225273132,
142
+ "kl": 0.007823562622070313,
143
+ "learning_rate": 2.3263454721781537e-06,
144
+ "loss": 0.042,
145
+ "reward": 0.8042411088943482,
146
+ "reward_std": 0.17801097435876728,
147
+ "rewards/accuracy_reward": 0.8042411088943482,
148
+ "rewards/format_reward": 0.0,
149
+ "step": 45
150
+ },
151
+ {
152
+ "clip_ratio": 0.0,
153
+ "completion_length": 582.4127494812012,
154
  "epoch": 0.42643923240938164,
155
+ "grad_norm": 0.13581737875938416,
156
+ "kl": 0.005602645874023438,
157
+ "learning_rate": 2.1305358424643485e-06,
158
+ "loss": 0.04,
159
+ "reward": 0.7600446760654449,
160
+ "reward_std": 0.18583385180681944,
161
+ "rewards/accuracy_reward": 0.7600446760654449,
162
  "rewards/format_reward": 0.0,
163
+ "step": 50
164
  },
165
  {
166
  "clip_ratio": 0.0,
167
+ "completion_length": 570.5196685791016,
168
+ "epoch": 0.4690831556503198,
169
+ "grad_norm": 0.14316147565841675,
170
+ "kl": 0.006391143798828125,
171
+ "learning_rate": 1.9206410839590043e-06,
172
+ "loss": 0.0253,
173
+ "reward": 0.770982176065445,
174
+ "reward_std": 0.1873089073225856,
175
+ "rewards/accuracy_reward": 0.770982176065445,
176
+ "rewards/format_reward": 0.0,
177
+ "step": 55
178
+ },
179
+ {
180
+ "clip_ratio": 0.0,
181
+ "completion_length": 597.8089546203613,
182
  "epoch": 0.511727078891258,
183
+ "grad_norm": 0.13278549909591675,
184
+ "kl": 0.006211090087890625,
185
+ "learning_rate": 1.7013498987264833e-06,
186
+ "loss": 0.0323,
187
+ "reward": 0.7582589626312256,
188
+ "reward_std": 0.19469668436795473,
189
+ "rewards/accuracy_reward": 0.7582589626312256,
190
  "rewards/format_reward": 0.0,
191
+ "step": 60
192
+ },
193
+ {
194
+ "clip_ratio": 0.0,
195
+ "completion_length": 570.72748336792,
196
+ "epoch": 0.5543710021321961,
197
+ "grad_norm": 0.11964963376522064,
198
+ "kl": 0.0067108154296875,
199
+ "learning_rate": 1.4775608894771048e-06,
200
+ "loss": 0.0285,
201
+ "reward": 0.7604911118745804,
202
+ "reward_std": 0.17904917486011981,
203
+ "rewards/accuracy_reward": 0.7604911118745804,
204
+ "rewards/format_reward": 0.0,
205
+ "step": 65
206
  },
207
  {
208
  "clip_ratio": 0.0,
209
+ "completion_length": 583.9893104553223,
210
  "epoch": 0.5970149253731343,
211
+ "grad_norm": 0.13020634651184082,
212
+ "kl": 0.005590057373046875,
213
+ "learning_rate": 1.2542731328772936e-06,
214
+ "loss": 0.0335,
215
+ "reward": 0.7493303909897804,
216
+ "reward_std": 0.19070985857397318,
217
+ "rewards/accuracy_reward": 0.7493303909897804,
218
  "rewards/format_reward": 0.0,
219
+ "step": 70
220
+ },
221
+ {
222
+ "clip_ratio": 0.0,
223
+ "completion_length": 552.5370803833008,
224
+ "epoch": 0.6396588486140725,
225
+ "grad_norm": 0.193593829870224,
226
+ "kl": 0.00690155029296875,
227
+ "learning_rate": 1.036474508437579e-06,
228
+ "loss": 0.0293,
229
+ "reward": 0.7955357491970062,
230
+ "reward_std": 0.17065229499712586,
231
+ "rewards/accuracy_reward": 0.7955357491970062,
232
+ "rewards/format_reward": 0.0,
233
+ "step": 75
234
  },
235
  {
236
  "clip_ratio": 0.0,
237
+ "completion_length": 574.2089538574219,
238
  "epoch": 0.6823027718550106,
239
+ "grad_norm": 0.26767051219940186,
240
+ "kl": 0.006514739990234375,
241
+ "learning_rate": 8.290302775265509e-07,
242
+ "loss": 0.0359,
243
+ "reward": 0.7752232506871224,
244
+ "reward_std": 0.17853977270424365,
245
+ "rewards/accuracy_reward": 0.7752232506871224,
246
  "rewards/format_reward": 0.0,
247
+ "step": 80
248
+ },
249
+ {
250
+ "clip_ratio": 0.0,
251
+ "completion_length": 587.2473434448242,
252
+ "epoch": 0.7249466950959488,
253
+ "grad_norm": 0.14498883485794067,
254
+ "kl": 0.008393096923828124,
255
+ "learning_rate": 6.3657440147149e-07,
256
+ "loss": 0.0378,
257
+ "reward": 0.77857146859169,
258
+ "reward_std": 0.1991961758583784,
259
+ "rewards/accuracy_reward": 0.77857146859169,
260
+ "rewards/format_reward": 0.0,
261
+ "step": 85
262
  },
263
  {
264
  "clip_ratio": 0.0,
265
+ "completion_length": 571.0614074707031,
266
  "epoch": 0.767590618336887,
267
+ "grad_norm": 0.3927496671676636,
268
+ "kl": 0.00813751220703125,
269
+ "learning_rate": 4.63406026519703e-07,
270
+ "loss": 0.0197,
271
+ "reward": 0.753348246216774,
272
+ "reward_std": 0.18298904253169895,
273
+ "rewards/accuracy_reward": 0.753348246216774,
274
  "rewards/format_reward": 0.0,
275
+ "step": 90
276
  },
277
  {
278
  "clip_ratio": 0.0,
279
+ "completion_length": 579.4897560119629,
280
+ "epoch": 0.8102345415778252,
281
+ "grad_norm": 0.15556038916110992,
282
+ "kl": 0.007627105712890625,
283
+ "learning_rate": 3.133934480154885e-07,
284
+ "loss": 0.0348,
285
+ "reward": 0.7566964641213417,
286
+ "reward_std": 0.20147906215861439,
287
+ "rewards/accuracy_reward": 0.7566964641213417,
288
+ "rewards/format_reward": 0.0,
289
+ "step": 95
290
+ },
291
+ {
292
  "epoch": 0.8528784648187633,
293
+ "grad_norm": 0.397614985704422,
294
+ "learning_rate": 1.8988769907430552e-07,
295
+ "loss": 0.0301,
296
+ "step": 100
297
+ },
298
+ {
299
+ "epoch": 0.8528784648187633,
300
+ "eval_clip_ratio": 0.0,
301
+ "eval_completion_length": 558.2626213378907,
302
+ "eval_kl": 0.02655439453125,
303
+ "eval_loss": 0.01617129147052765,
304
+ "eval_reward": 0.6867000318527222,
305
+ "eval_reward_std": 0.22212331930994988,
306
+ "eval_rewards/accuracy_reward": 0.6867000318527222,
307
+ "eval_rewards/format_reward": 0.0,
308
+ "eval_runtime": 17227.6498,
309
+ "eval_samples_per_second": 0.29,
310
+ "eval_steps_per_second": 0.003,
311
+ "step": 100
312
+ },
313
+ {
314
+ "clip_ratio": 0.0,
315
+ "completion_length": 574.3294898986817,
316
+ "epoch": 0.8955223880597015,
317
+ "grad_norm": 0.7429274916648865,
318
+ "kl": 0.007941818237304688,
319
+ "learning_rate": 9.564769404039419e-08,
320
+ "loss": 0.0334,
321
+ "reward": 0.7816964637488126,
322
+ "reward_std": 0.18899818495847284,
323
+ "rewards/accuracy_reward": 0.7816964637488126,
324
  "rewards/format_reward": 0.0,
325
+ "step": 105
326
  },
327
  {
328
  "clip_ratio": 0.0,
329
+ "completion_length": 574.8442253112793,
330
  "epoch": 0.9381663113006397,
331
+ "grad_norm": 0.1576305478811264,
332
+ "kl": 0.00836029052734375,
333
+ "learning_rate": 3.277859889929147e-08,
334
+ "loss": 0.0323,
335
+ "reward": 0.8006696775555611,
336
+ "reward_std": 0.19058242589235305,
337
+ "rewards/accuracy_reward": 0.8006696775555611,
338
  "rewards/format_reward": 0.0,
339
+ "step": 110
340
+ },
341
+ {
342
+ "clip_ratio": 0.0,
343
+ "completion_length": 558.2982414245605,
344
+ "epoch": 0.9808102345415778,
345
+ "grad_norm": 0.16698426008224487,
346
+ "kl": 0.0088043212890625,
347
+ "learning_rate": 2.684805348397268e-09,
348
+ "loss": 0.0311,
349
+ "reward": 0.7834821820259095,
350
+ "reward_std": 0.17699794424697757,
351
+ "rewards/accuracy_reward": 0.7834821820259095,
352
+ "rewards/format_reward": 0.0,
353
+ "step": 115
354
  },
355
  {
356
  "clip_ratio": 0.0,
357
+ "completion_length": 580.5032634735107,
358
+ "epoch": 0.997867803837953,
359
+ "kl": 0.008594512939453125,
360
+ "reward": 0.7589286100119352,
361
+ "reward_std": 0.21703570941463113,
362
+ "rewards/accuracy_reward": 0.7589286100119352,
363
  "rewards/format_reward": 0.0,
364
+ "step": 117,
365
  "total_flos": 0.0,
366
+ "train_loss": 0.03906208295056708,
367
+ "train_runtime": 48868.7895,
368
+ "train_samples_per_second": 0.153,
369
+ "train_steps_per_second": 0.002
370
  }
371
  ],
372
  "logging_steps": 5,
373
+ "max_steps": 117,
374
  "num_input_tokens_seen": 0,
375
  "num_train_epochs": 1,
376
  "save_steps": 500,
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:3af8e5e514d734b23d1d77c7af762c461e4995da84b8ea65e1f800d79e7e4ab6
3
  size 7992
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c60b555cc7f9b788a7920f8f5e261d1fa93487fc7555e9f8c3ee4670ba6d2f20
3
  size 7992