End of training

README.md

@@ -16,9 +16,9 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model was trained from scratch on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 419.5198
-- Wer: 0.7614
-- Cer: 0.3089
+- Loss: 271.5626
+- Wer: 0.4858
+- Cer: 0.1540
 
 ## Model description
 

all_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 12.0,
+    "eval_cer": 0.15402855308007707,
+    "eval_loss": 271.56256103515625,
+    "eval_runtime": 26.2754,
+    "eval_samples": 890,
+    "eval_samples_per_second": 33.872,
+    "eval_steps_per_second": 4.263,
+    "eval_wer": 0.48579545454545453,
+    "total_flos": 3.6380056389804335e+19,
+    "train_loss": 870.7509072951121,
+    "train_runtime": 30399.5697,
+    "train_samples": 58040,
+    "train_samples_per_second": 114.554,
+    "train_steps_per_second": 7.161
+}

eval--in domain_results.json

@@ -0,0 +1,10 @@
+{
+    "epoch": 12.0,
+    "eval_cer": 0.15402855308007707,
+    "eval_loss": 271.56256103515625,
+    "eval_runtime": 26.2754,
+    "eval_samples": 890,
+    "eval_samples_per_second": 33.872,
+    "eval_steps_per_second": 4.263,
+    "eval_wer": 0.48579545454545453
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 12.0,
+    "total_flos": 3.6380056389804335e+19,
+    "train_loss": 870.7509072951121,
+    "train_runtime": 30399.5697,
+    "train_samples": 58040,
+    "train_samples_per_second": 114.554,
+    "train_steps_per_second": 7.161
+}

trainer_state.json

@@ -0,0 +1,255 @@
+{
+  "best_metric": 0.4881628787878788,
+  "best_model_checkpoint": "/scratch/elec/puhe/p/palp3/sami_ASR/base_model_output/wav2vec2-base-sami-cont-pt-22k/outputs/shuff_100_no_idv/wr20/checkpoint-7256",
+  "epoch": 12.0,
+  "eval_steps": 500,
+  "global_step": 43536,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 1.0,
+      "grad_norm": 3081.265380859375,
+      "learning_rate": 4.1482910694597574e-05,
+      "loss": 2875.4683,
+      "step": 3628
+    },
+    {
+      "epoch": 1.0,
+      "eval_cer": 0.17331917205947736,
+      "eval_loss": 287.1173400878906,
+      "eval_runtime": 25.5716,
+      "eval_samples_per_second": 34.804,
+      "eval_steps_per_second": 4.38,
+      "eval_wer": 0.5699179292929293,
+      "step": 3628
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 591.99072265625,
+      "learning_rate": 8.313809261300991e-05,
+      "loss": 758.2716,
+      "step": 7256
+    },
+    {
+      "epoch": 2.0,
+      "eval_cer": 0.1550906486192758,
+      "eval_loss": 272.05938720703125,
+      "eval_runtime": 24.8059,
+      "eval_samples_per_second": 35.879,
+      "eval_steps_per_second": 4.515,
+      "eval_wer": 0.4881628787878788,
+      "step": 7256
+    },
+    {
+      "epoch": 3.0,
+      "grad_norm": 477.86785888671875,
+      "learning_rate": 0.0001248047592796766,
+      "loss": 644.8381,
+      "step": 10884
+    },
+    {
+      "epoch": 3.0,
+      "eval_cer": 0.1729486736155708,
+      "eval_loss": 290.3243713378906,
+      "eval_runtime": 25.3553,
+      "eval_samples_per_second": 35.101,
+      "eval_steps_per_second": 4.417,
+      "eval_wer": 0.506155303030303,
+      "step": 10884
+    },
+    {
+      "epoch": 4.0,
+      "grad_norm": 748.2718505859375,
+      "learning_rate": 0.00016645994119808893,
+      "loss": 609.5405,
+      "step": 14512
+    },
+    {
+      "epoch": 4.0,
+      "eval_cer": 0.18673121572889395,
+      "eval_loss": 257.9621887207031,
+      "eval_runtime": 24.6972,
+      "eval_samples_per_second": 36.037,
+      "eval_steps_per_second": 4.535,
+      "eval_wer": 0.5173611111111112,
+      "step": 14512
+    },
+    {
+      "epoch": 5.0,
+      "grad_norm": 547.5361938476562,
+      "learning_rate": 0.0002081151231165013,
+      "loss": 606.3339,
+      "step": 18140
+    },
+    {
+      "epoch": 5.0,
+      "eval_cer": 0.22798004248382156,
+      "eval_loss": 296.2996520996094,
+      "eval_runtime": 26.0793,
+      "eval_samples_per_second": 34.127,
+      "eval_steps_per_second": 4.295,
+      "eval_wer": 0.5667613636363636,
+      "step": 18140
+    },
+    {
+      "epoch": 6.0,
+      "grad_norm": 480.7348937988281,
+      "learning_rate": 0.00024977030503491363,
+      "loss": 619.5554,
+      "step": 21768
+    },
+    {
+      "epoch": 6.0,
+      "eval_cer": 0.2187916810749395,
+      "eval_loss": 324.2591247558594,
+      "eval_runtime": 24.5568,
+      "eval_samples_per_second": 36.243,
+      "eval_steps_per_second": 4.561,
+      "eval_wer": 0.5901199494949495,
+      "step": 21768
+    },
+    {
+      "epoch": 7.0,
+      "grad_norm": 429.8402404785156,
+      "learning_rate": 0.00029141400220507167,
+      "loss": 646.4168,
+      "step": 25396
+    },
+    {
+      "epoch": 7.0,
+      "eval_cer": 0.24396087536432348,
+      "eval_loss": 341.2834167480469,
+      "eval_runtime": 25.9793,
+      "eval_samples_per_second": 34.258,
+      "eval_steps_per_second": 4.311,
+      "eval_wer": 0.5953282828282829,
+      "step": 25396
+    },
+    {
+      "epoch": 8.0,
+      "grad_norm": 477.2936096191406,
+      "learning_rate": 0.0003330576993752297,
+      "loss": 667.3437,
+      "step": 29024
+    },
+    {
+      "epoch": 8.0,
+      "eval_cer": 0.22546065306525712,
+      "eval_loss": 361.0743713378906,
+      "eval_runtime": 25.9631,
+      "eval_samples_per_second": 34.279,
+      "eval_steps_per_second": 4.314,
+      "eval_wer": 0.6283143939393939,
+      "step": 29024
+    },
+    {
+      "epoch": 9.0,
+      "grad_norm": 485.8271179199219,
+      "learning_rate": 0.00037471288129364204,
+      "loss": 693.7482,
+      "step": 32652
+    },
+    {
+      "epoch": 9.0,
+      "eval_cer": 0.268537272143457,
+      "eval_loss": 364.4578857421875,
+      "eval_runtime": 24.4432,
+      "eval_samples_per_second": 36.411,
+      "eval_steps_per_second": 4.582,
+      "eval_wer": 0.6597222222222222,
+      "step": 32652
+    },
+    {
+      "epoch": 10.0,
+      "grad_norm": 167.16514587402344,
+      "learning_rate": 0.0004163450937155457,
+      "loss": 733.3208,
+      "step": 36280
+    },
+    {
+      "epoch": 10.0,
+      "eval_cer": 0.28022032307464306,
+      "eval_loss": 409.5367736816406,
+      "eval_runtime": 26.9757,
+      "eval_samples_per_second": 32.993,
+      "eval_steps_per_second": 4.152,
+      "eval_wer": 0.7001262626262627,
+      "step": 36280
+    },
+    {
+      "epoch": 11.0,
+      "grad_norm": 588.0139770507812,
+      "learning_rate": 0.0004580002756339581,
+      "loss": 780.5633,
+      "step": 39908
+    },
+    {
+      "epoch": 11.0,
+      "eval_cer": 0.2849380032603863,
+      "eval_loss": 410.3908386230469,
+      "eval_runtime": 24.8976,
+      "eval_samples_per_second": 35.746,
+      "eval_steps_per_second": 4.498,
+      "eval_wer": 0.7154356060606061,
+      "step": 39908
+    },
+    {
+      "epoch": 12.0,
+      "grad_norm": 167.97271728515625,
+      "learning_rate": 0.0004996669423006247,
+      "loss": 813.6104,
+      "step": 43536
+    },
+    {
+      "epoch": 12.0,
+      "eval_cer": 0.3088722027367485,
+      "eval_loss": 419.51983642578125,
+      "eval_runtime": 27.7709,
+      "eval_samples_per_second": 32.048,
+      "eval_steps_per_second": 4.033,
+      "eval_wer": 0.7613636363636364,
+      "step": 43536
+    },
+    {
+      "epoch": 12.0,
+      "step": 43536,
+      "total_flos": 3.6380056389804335e+19,
+      "train_loss": 870.7509072951121,
+      "train_runtime": 30399.5697,
+      "train_samples_per_second": 114.554,
+      "train_steps_per_second": 7.161
+    }
+  ],
+  "logging_steps": 500,
+  "max_steps": 217680,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 60,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "EarlyStoppingCallback": {
+      "args": {
+        "early_stopping_patience": 10,
+        "early_stopping_threshold": 0.0
+      },
+      "attributes": {
+        "early_stopping_patience_counter": 10
+      }
+    },
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.6380056389804335e+19,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}