Priyanship commited on
Commit
419d0a3
·
verified ·
1 Parent(s): fb251a8

End of training

Browse files
README.md CHANGED
@@ -16,7 +16,7 @@ should probably proofread and complete it, then remove this comment. -->
16
 
17
  This model was trained from scratch on an unknown dataset.
18
  It achieves the following results on the evaluation set:
19
- - Loss: 1141.5001
20
  - Wer: 1.0
21
  - Cer: 1.0
22
 
 
16
 
17
  This model was trained from scratch on an unknown dataset.
18
  It achieves the following results on the evaluation set:
19
+ - Loss: 1144.6145
20
  - Wer: 1.0
21
  - Cer: 1.0
22
 
all_results.json ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 20.0,
3
+ "eval_cer": 1.0,
4
+ "eval_loss": 1144.614501953125,
5
+ "eval_runtime": 23.5534,
6
+ "eval_samples": 890,
7
+ "eval_samples_per_second": 37.786,
8
+ "eval_steps_per_second": 4.755,
9
+ "eval_wer": 1.0,
10
+ "total_flos": 3.1981440469898e+19,
11
+ "train_loss": 4410.907295081967,
12
+ "train_runtime": 25843.4041,
13
+ "train_samples": 24395,
14
+ "train_samples_per_second": 18.879,
15
+ "train_steps_per_second": 1.18
16
+ }
eval--1hr test set_results.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 20.0,
3
+ "eval_cer": 1.0,
4
+ "eval_loss": 1144.614501953125,
5
+ "eval_runtime": 23.5534,
6
+ "eval_samples": 890,
7
+ "eval_samples_per_second": 37.786,
8
+ "eval_steps_per_second": 4.755,
9
+ "eval_wer": 1.0
10
+ }
train_results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 20.0,
3
+ "total_flos": 3.1981440469898e+19,
4
+ "train_loss": 4410.907295081967,
5
+ "train_runtime": 25843.4041,
6
+ "train_samples": 24395,
7
+ "train_samples_per_second": 18.879,
8
+ "train_steps_per_second": 1.18
9
+ }
trainer_state.json ADDED
@@ -0,0 +1,382 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": 1.0,
3
+ "best_model_checkpoint": "/scratch/elec/puhe/p/palp3/sami_ASR/base_model_output/base-sami-cont-pt-22k/mixlabels/rerun/ep20/outputs/checkpoint-1525",
4
+ "epoch": 20.0,
5
+ "eval_steps": 500,
6
+ "global_step": 30500,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 1.0,
13
+ "grad_norm": 599.1070556640625,
14
+ "learning_rate": 9.908196721311476e-05,
15
+ "loss": 5425.9882,
16
+ "step": 1525
17
+ },
18
+ {
19
+ "epoch": 1.0,
20
+ "eval_cer": 1.0,
21
+ "eval_loss": 1144.5670166015625,
22
+ "eval_runtime": 24.5176,
23
+ "eval_samples_per_second": 36.3,
24
+ "eval_steps_per_second": 4.568,
25
+ "eval_wer": 1.0,
26
+ "step": 1525
27
+ },
28
+ {
29
+ "epoch": 2.0,
30
+ "grad_norm": 729.23193359375,
31
+ "learning_rate": 0.00019908196721311476,
32
+ "loss": 4416.5207,
33
+ "step": 3050
34
+ },
35
+ {
36
+ "epoch": 2.0,
37
+ "eval_cer": 1.0,
38
+ "eval_loss": 1142.759521484375,
39
+ "eval_runtime": 26.3683,
40
+ "eval_samples_per_second": 33.753,
41
+ "eval_steps_per_second": 4.248,
42
+ "eval_wer": 1.0,
43
+ "step": 3050
44
+ },
45
+ {
46
+ "epoch": 3.0,
47
+ "grad_norm": 611.4544677734375,
48
+ "learning_rate": 0.00029908196721311475,
49
+ "loss": 4474.501,
50
+ "step": 4575
51
+ },
52
+ {
53
+ "epoch": 3.0,
54
+ "eval_cer": 1.0,
55
+ "eval_loss": 1144.2608642578125,
56
+ "eval_runtime": 25.6743,
57
+ "eval_samples_per_second": 34.665,
58
+ "eval_steps_per_second": 4.362,
59
+ "eval_wer": 1.0,
60
+ "step": 4575
61
+ },
62
+ {
63
+ "epoch": 4.0,
64
+ "grad_norm": 435.5348815917969,
65
+ "learning_rate": 0.00039901639344262297,
66
+ "loss": 4499.2403,
67
+ "step": 6100
68
+ },
69
+ {
70
+ "epoch": 4.0,
71
+ "eval_cer": 1.0,
72
+ "eval_loss": 1163.198486328125,
73
+ "eval_runtime": 24.0072,
74
+ "eval_samples_per_second": 37.072,
75
+ "eval_steps_per_second": 4.665,
76
+ "eval_wer": 1.0,
77
+ "step": 6100
78
+ },
79
+ {
80
+ "epoch": 5.0,
81
+ "grad_norm": 500.39129638671875,
82
+ "learning_rate": 0.0004989508196721312,
83
+ "loss": 4492.3184,
84
+ "step": 7625
85
+ },
86
+ {
87
+ "epoch": 5.0,
88
+ "eval_cer": 1.0,
89
+ "eval_loss": 1145.3702392578125,
90
+ "eval_runtime": 23.8651,
91
+ "eval_samples_per_second": 37.293,
92
+ "eval_steps_per_second": 4.693,
93
+ "eval_wer": 1.0,
94
+ "step": 7625
95
+ },
96
+ {
97
+ "epoch": 6.0,
98
+ "grad_norm": 480.6201171875,
99
+ "learning_rate": 0.000467016393442623,
100
+ "loss": 4439.5584,
101
+ "step": 9150
102
+ },
103
+ {
104
+ "epoch": 6.0,
105
+ "eval_cer": 1.0,
106
+ "eval_loss": 1145.4093017578125,
107
+ "eval_runtime": 26.4383,
108
+ "eval_samples_per_second": 33.663,
109
+ "eval_steps_per_second": 4.236,
110
+ "eval_wer": 1.0,
111
+ "step": 9150
112
+ },
113
+ {
114
+ "epoch": 7.0,
115
+ "grad_norm": 496.0490417480469,
116
+ "learning_rate": 0.00043370491803278687,
117
+ "loss": 4400.9305,
118
+ "step": 10675
119
+ },
120
+ {
121
+ "epoch": 7.0,
122
+ "eval_cer": 1.0,
123
+ "eval_loss": 1144.0640869140625,
124
+ "eval_runtime": 25.1926,
125
+ "eval_samples_per_second": 35.328,
126
+ "eval_steps_per_second": 4.446,
127
+ "eval_wer": 1.0,
128
+ "step": 10675
129
+ },
130
+ {
131
+ "epoch": 8.0,
132
+ "grad_norm": 500.4902038574219,
133
+ "learning_rate": 0.0004003715846994536,
134
+ "loss": 4388.3134,
135
+ "step": 12200
136
+ },
137
+ {
138
+ "epoch": 8.0,
139
+ "eval_cer": 1.0,
140
+ "eval_loss": 1142.6368408203125,
141
+ "eval_runtime": 25.9206,
142
+ "eval_samples_per_second": 34.336,
143
+ "eval_steps_per_second": 4.321,
144
+ "eval_wer": 1.0,
145
+ "step": 12200
146
+ },
147
+ {
148
+ "epoch": 9.0,
149
+ "grad_norm": 519.6370239257812,
150
+ "learning_rate": 0.00036706010928961745,
151
+ "loss": 4337.4364,
152
+ "step": 13725
153
+ },
154
+ {
155
+ "epoch": 9.0,
156
+ "eval_cer": 1.0,
157
+ "eval_loss": 1141.1510009765625,
158
+ "eval_runtime": 25.2158,
159
+ "eval_samples_per_second": 35.295,
160
+ "eval_steps_per_second": 4.442,
161
+ "eval_wer": 1.0,
162
+ "step": 13725
163
+ },
164
+ {
165
+ "epoch": 10.0,
166
+ "grad_norm": 622.8638305664062,
167
+ "learning_rate": 0.00033372677595628415,
168
+ "loss": 4328.8672,
169
+ "step": 15250
170
+ },
171
+ {
172
+ "epoch": 10.0,
173
+ "eval_cer": 1.0,
174
+ "eval_loss": 1140.218017578125,
175
+ "eval_runtime": 24.9601,
176
+ "eval_samples_per_second": 35.657,
177
+ "eval_steps_per_second": 4.487,
178
+ "eval_wer": 1.0,
179
+ "step": 15250
180
+ },
181
+ {
182
+ "epoch": 11.0,
183
+ "grad_norm": 536.9533081054688,
184
+ "learning_rate": 0.00030041530054644814,
185
+ "loss": 4315.4475,
186
+ "step": 16775
187
+ },
188
+ {
189
+ "epoch": 11.0,
190
+ "eval_cer": 1.0,
191
+ "eval_loss": 1140.9918212890625,
192
+ "eval_runtime": 24.0596,
193
+ "eval_samples_per_second": 36.991,
194
+ "eval_steps_per_second": 4.655,
195
+ "eval_wer": 1.0,
196
+ "step": 16775
197
+ },
198
+ {
199
+ "epoch": 12.0,
200
+ "grad_norm": 479.0085754394531,
201
+ "learning_rate": 0.0002671256830601093,
202
+ "loss": 4340.9885,
203
+ "step": 18300
204
+ },
205
+ {
206
+ "epoch": 12.0,
207
+ "eval_cer": 1.0,
208
+ "eval_loss": 1141.5726318359375,
209
+ "eval_runtime": 24.2656,
210
+ "eval_samples_per_second": 36.677,
211
+ "eval_steps_per_second": 4.616,
212
+ "eval_wer": 1.0,
213
+ "step": 18300
214
+ },
215
+ {
216
+ "epoch": 13.0,
217
+ "grad_norm": 435.52410888671875,
218
+ "learning_rate": 0.00023379234972677597,
219
+ "loss": 4303.5944,
220
+ "step": 19825
221
+ },
222
+ {
223
+ "epoch": 13.0,
224
+ "eval_cer": 1.0,
225
+ "eval_loss": 1140.975830078125,
226
+ "eval_runtime": 25.0082,
227
+ "eval_samples_per_second": 35.588,
228
+ "eval_steps_per_second": 4.479,
229
+ "eval_wer": 1.0,
230
+ "step": 19825
231
+ },
232
+ {
233
+ "epoch": 14.0,
234
+ "grad_norm": 576.5276489257812,
235
+ "learning_rate": 0.00020045901639344265,
236
+ "loss": 4305.4659,
237
+ "step": 21350
238
+ },
239
+ {
240
+ "epoch": 14.0,
241
+ "eval_cer": 1.0,
242
+ "eval_loss": 1141.2747802734375,
243
+ "eval_runtime": 24.5459,
244
+ "eval_samples_per_second": 36.259,
245
+ "eval_steps_per_second": 4.563,
246
+ "eval_wer": 1.0,
247
+ "step": 21350
248
+ },
249
+ {
250
+ "epoch": 15.0,
251
+ "grad_norm": 485.7404479980469,
252
+ "learning_rate": 0.00016714754098360658,
253
+ "loss": 4296.8341,
254
+ "step": 22875
255
+ },
256
+ {
257
+ "epoch": 15.0,
258
+ "eval_cer": 1.0,
259
+ "eval_loss": 1140.5506591796875,
260
+ "eval_runtime": 27.5372,
261
+ "eval_samples_per_second": 32.32,
262
+ "eval_steps_per_second": 4.067,
263
+ "eval_wer": 1.0,
264
+ "step": 22875
265
+ },
266
+ {
267
+ "epoch": 16.0,
268
+ "grad_norm": 455.784423828125,
269
+ "learning_rate": 0.00013381420765027323,
270
+ "loss": 4293.78,
271
+ "step": 24400
272
+ },
273
+ {
274
+ "epoch": 16.0,
275
+ "eval_cer": 1.0,
276
+ "eval_loss": 1141.3170166015625,
277
+ "eval_runtime": 23.8888,
278
+ "eval_samples_per_second": 37.256,
279
+ "eval_steps_per_second": 4.688,
280
+ "eval_wer": 1.0,
281
+ "step": 24400
282
+ },
283
+ {
284
+ "epoch": 17.0,
285
+ "grad_norm": 530.2776489257812,
286
+ "learning_rate": 0.00010050273224043716,
287
+ "loss": 4291.8813,
288
+ "step": 25925
289
+ },
290
+ {
291
+ "epoch": 17.0,
292
+ "eval_cer": 1.0,
293
+ "eval_loss": 1140.3837890625,
294
+ "eval_runtime": 25.3819,
295
+ "eval_samples_per_second": 35.064,
296
+ "eval_steps_per_second": 4.413,
297
+ "eval_wer": 1.0,
298
+ "step": 25925
299
+ },
300
+ {
301
+ "epoch": 18.0,
302
+ "grad_norm": 510.512451171875,
303
+ "learning_rate": 6.716939890710383e-05,
304
+ "loss": 4291.3587,
305
+ "step": 27450
306
+ },
307
+ {
308
+ "epoch": 18.0,
309
+ "eval_cer": 1.0,
310
+ "eval_loss": 1141.1961669921875,
311
+ "eval_runtime": 24.3571,
312
+ "eval_samples_per_second": 36.54,
313
+ "eval_steps_per_second": 4.598,
314
+ "eval_wer": 1.0,
315
+ "step": 27450
316
+ },
317
+ {
318
+ "epoch": 19.0,
319
+ "grad_norm": 453.8408508300781,
320
+ "learning_rate": 3.383606557377049e-05,
321
+ "loss": 4288.1016,
322
+ "step": 28975
323
+ },
324
+ {
325
+ "epoch": 19.0,
326
+ "eval_cer": 1.0,
327
+ "eval_loss": 1140.9141845703125,
328
+ "eval_runtime": 23.8605,
329
+ "eval_samples_per_second": 37.3,
330
+ "eval_steps_per_second": 4.694,
331
+ "eval_wer": 1.0,
332
+ "step": 28975
333
+ },
334
+ {
335
+ "epoch": 20.0,
336
+ "grad_norm": 636.82763671875,
337
+ "learning_rate": 5.245901639344263e-07,
338
+ "loss": 4287.0193,
339
+ "step": 30500
340
+ },
341
+ {
342
+ "epoch": 20.0,
343
+ "eval_cer": 1.0,
344
+ "eval_loss": 1141.5001220703125,
345
+ "eval_runtime": 26.8787,
346
+ "eval_samples_per_second": 33.112,
347
+ "eval_steps_per_second": 4.167,
348
+ "eval_wer": 1.0,
349
+ "step": 30500
350
+ },
351
+ {
352
+ "epoch": 20.0,
353
+ "step": 30500,
354
+ "total_flos": 3.1981440469898e+19,
355
+ "train_loss": 4410.907295081967,
356
+ "train_runtime": 25843.4041,
357
+ "train_samples_per_second": 18.879,
358
+ "train_steps_per_second": 1.18
359
+ }
360
+ ],
361
+ "logging_steps": 500,
362
+ "max_steps": 30500,
363
+ "num_input_tokens_seen": 0,
364
+ "num_train_epochs": 20,
365
+ "save_steps": 500,
366
+ "stateful_callbacks": {
367
+ "TrainerControl": {
368
+ "args": {
369
+ "should_epoch_stop": false,
370
+ "should_evaluate": false,
371
+ "should_log": false,
372
+ "should_save": true,
373
+ "should_training_stop": true
374
+ },
375
+ "attributes": {}
376
+ }
377
+ },
378
+ "total_flos": 3.1981440469898e+19,
379
+ "train_batch_size": 16,
380
+ "trial_name": null,
381
+ "trial_params": null
382
+ }