Training in progress, step 96000, checkpoint

checkpoint-96000/config.json

@@ -1,32 +1,28 @@
 {
-  "activation_function": "gelu_new",
   "architectures": [
-    "GPT2LMHeadModel"
+    "RobertaForMaskedLM"
   ],
-  "attn_pdrop": 0.1,
+  "attention_probs_dropout_prob": 0.1,
   "bos_token_id": 0,
-  "embd_pdrop": 0.1,
+  "classifier_dropout": null,
   "eos_token_id": 1,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 256,
   "initializer_range": 0.02,
-  "layer_norm_epsilon": 1e-05,
-  "model_type": "gpt2",
-  "n_ctx": 512,
-  "n_embd": 256,
-  "n_head": 8,
-  "n_inner": 2048,
-  "n_layer": 8,
-  "n_positions": 512,
-  "reorder_and_upcast_attn": false,
-  "resid_pdrop": 0.1,
-  "scale_attn_by_inverse_layer_idx": false,
-  "scale_attn_weights": true,
-  "summary_activation": null,
-  "summary_first_dropout": 0.1,
-  "summary_proj_to_labels": true,
-  "summary_type": "cls_index",
-  "summary_use_proj": true,
+  "intermediate_size": 2048,
+  "layer_norm_eps": 1e-05,
+  "mask_token_id": 4,
+  "max_position_embeddings": 512,
+  "model_type": "roberta",
+  "num_attention_heads": 8,
+  "num_hidden_layers": 8,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "tie_word_embeddings": false,
   "torch_dtype": "float32",
   "transformers_version": "4.45.2",
+  "type_vocab_size": 2,
   "use_cache": true,
   "vocab_size": 8192
 }

checkpoint-96000/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e682090e2c325be41eddadf1de552e16b52f85d294c84f063919ea7e1d41c16a
-size 51007160
+oid sha256:4c8f0419c425bb1f965f3bd40a3b54c8ef36a9f02cbb2ee49b0059ff8f16ef49
+size 59702184

checkpoint-96000/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:be8404da684b6505bc1266a417b03741f1a792c7e39e94a80e9e4c23dda46bc6
-size 102078202
+oid sha256:5cf6470cc4b5915e57baaccc4f7eba5247a0422b7d20c00601b0e3838f770b81
+size 119488058

checkpoint-96000/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:64affa433595876c1dc9cd3a758ebc7296308f21cf5908b0f0c9701c7a74a221
+oid sha256:42164cb038094c4d755f0ac429a998fb7f9bdafd3018005da4a39330e143524b
 size 14244

checkpoint-96000/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5db4902f0faa4194790ec152f388561eabd6c050748b13f2a0c329d4ce273e92
+oid sha256:f3af3f899871d82259b68c6848039a1b08e4b7ff0446a8cdc43807b1313f1901
 size 1000

checkpoint-96000/special_tokens_map.json

@@ -23,7 +23,13 @@
     "rstrip": false,
     "single_word": false
   },
-  "pad_token": "</s>",
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
   "unk_token": {
     "content": "<unk>",
     "lstrip": false,

checkpoint-96000/tokenizer_config.json

@@ -57,7 +57,7 @@
   "eos_token": "</s>",
   "mask_token": "<mask>",
   "model_max_length": 128,
-  "pad_token": "</s>",
+  "pad_token": "<pad>",
   "tokenizer_class": "PreTrainedTokenizerFast",
   "unk_token": "<unk>"
 }

checkpoint-96000/trainer_state.json

@@ -1,6 +1,6 @@
 {
-  "best_metric": 3.307403087615967,
-  "best_model_checkpoint": "/home/p318482/babyLM_controlled/models_trained_last/fr_clm/fr_childes_42/checkpoint-36000",
+  "best_metric": 1.8838213682174683,
+  "best_model_checkpoint": "/home/p318482/babyLM_controlled/models_trained_last/fr_mlm/fr_childes_42/checkpoint-96000",
   "epoch": 120.0,
   "eval_steps": 2000,
   "global_step": 96000,
@@ -10,554 +10,554 @@
   "log_history": [
     {
       "epoch": 2.5,
-      "eval_loss": 6.572530269622803,
-      "eval_runtime": 2.3766,
-      "eval_samples_per_second": 868.883,
-      "eval_steps_per_second": 54.7,
+      "eval_loss": 7.232339382171631,
+      "eval_runtime": 3.1848,
+      "eval_samples_per_second": 648.395,
+      "eval_steps_per_second": 40.819,
       "step": 2000
     },
     {
       "epoch": 5.0,
-      "grad_norm": 1.6818779706954956,
+      "grad_norm": 1.1186400651931763,
       "learning_rate": 1e-05,
-      "loss": 6.5185,
+      "loss": 7.2283,
       "step": 4000
     },
     {
       "epoch": 5.0,
-      "eval_loss": 4.926427364349365,
-      "eval_runtime": 2.4953,
-      "eval_samples_per_second": 827.57,
-      "eval_steps_per_second": 52.099,
+      "eval_loss": 5.976784706115723,
+      "eval_runtime": 3.0375,
+      "eval_samples_per_second": 679.836,
+      "eval_steps_per_second": 42.798,
       "step": 4000
     },
     {
       "epoch": 7.5,
-      "eval_loss": 4.468189716339111,
-      "eval_runtime": 2.5391,
-      "eval_samples_per_second": 813.275,
-      "eval_steps_per_second": 51.199,
+      "eval_loss": 5.826813220977783,
+      "eval_runtime": 2.9574,
+      "eval_samples_per_second": 698.238,
+      "eval_steps_per_second": 43.957,
       "step": 6000
     },
     {
       "epoch": 10.0,
-      "grad_norm": 2.349691867828369,
+      "grad_norm": 1.8301379680633545,
       "learning_rate": 2e-05,
-      "loss": 4.3057,
+      "loss": 5.6892,
       "step": 8000
     },
     {
       "epoch": 10.0,
-      "eval_loss": 4.204324245452881,
-      "eval_runtime": 2.4223,
-      "eval_samples_per_second": 852.479,
-      "eval_steps_per_second": 53.667,
+      "eval_loss": 5.744414806365967,
+      "eval_runtime": 2.5747,
+      "eval_samples_per_second": 802.048,
+      "eval_steps_per_second": 50.492,
       "step": 8000
     },
     {
       "epoch": 12.5,
-      "eval_loss": 4.019620895385742,
-      "eval_runtime": 2.3623,
-      "eval_samples_per_second": 874.13,
-      "eval_steps_per_second": 55.03,
+      "eval_loss": 5.670751571655273,
+      "eval_runtime": 1.618,
+      "eval_samples_per_second": 1276.247,
+      "eval_steps_per_second": 80.345,
       "step": 10000
     },
     {
       "epoch": 15.0,
-      "grad_norm": 2.640118360519409,
-      "learning_rate": 2.9995e-05,
-      "loss": 3.8252,
+      "grad_norm": 2.46317982673645,
+      "learning_rate": 3e-05,
+      "loss": 5.4694,
       "step": 12000
     },
     {
       "epoch": 15.0,
-      "eval_loss": 3.879347562789917,
-      "eval_runtime": 2.4514,
-      "eval_samples_per_second": 842.389,
-      "eval_steps_per_second": 53.032,
+      "eval_loss": 5.57132625579834,
+      "eval_runtime": 2.872,
+      "eval_samples_per_second": 719.021,
+      "eval_steps_per_second": 45.265,
       "step": 12000
     },
     {
       "epoch": 17.5,
-      "eval_loss": 3.773747205734253,
-      "eval_runtime": 2.342,
-      "eval_samples_per_second": 881.708,
-      "eval_steps_per_second": 55.507,
+      "eval_loss": 5.526998043060303,
+      "eval_runtime": 3.0079,
+      "eval_samples_per_second": 686.536,
+      "eval_steps_per_second": 43.22,
       "step": 14000
     },
     {
       "epoch": 20.0,
-      "grad_norm": 2.4218180179595947,
-      "learning_rate": 3.999e-05,
-      "loss": 3.5527,
+      "grad_norm": 3.224332571029663,
+      "learning_rate": 4e-05,
+      "loss": 5.3091,
       "step": 16000
     },
     {
       "epoch": 20.0,
-      "eval_loss": 3.6714212894439697,
-      "eval_runtime": 2.448,
-      "eval_samples_per_second": 843.542,
-      "eval_steps_per_second": 53.104,
+      "eval_loss": 5.448328971862793,
+      "eval_runtime": 2.9966,
+      "eval_samples_per_second": 689.104,
+      "eval_steps_per_second": 43.382,
       "step": 16000
     },
     {
       "epoch": 22.5,
-      "eval_loss": 3.593899965286255,
-      "eval_runtime": 2.3722,
-      "eval_samples_per_second": 870.507,
-      "eval_steps_per_second": 54.802,
+      "eval_loss": 5.3926005363464355,
+      "eval_runtime": 3.0264,
+      "eval_samples_per_second": 682.332,
+      "eval_steps_per_second": 42.955,
       "step": 18000
     },
     {
       "epoch": 25.0,
-      "grad_norm": 2.246117115020752,
-      "learning_rate": 4.9985e-05,
-      "loss": 3.3505,
+      "grad_norm": 3.280613660812378,
+      "learning_rate": 5e-05,
+      "loss": 5.1707,
       "step": 20000
     },
     {
       "epoch": 25.0,
-      "eval_loss": 3.526674747467041,
-      "eval_runtime": 2.3702,
-      "eval_samples_per_second": 871.219,
-      "eval_steps_per_second": 54.847,
+      "eval_loss": 5.231536388397217,
+      "eval_runtime": 3.0403,
+      "eval_samples_per_second": 679.22,
+      "eval_steps_per_second": 42.76,
       "step": 20000
     },
     {
       "epoch": 27.5,
-      "eval_loss": 3.463704824447632,
-      "eval_runtime": 2.4208,
-      "eval_samples_per_second": 853.019,
-      "eval_steps_per_second": 53.701,
+      "eval_loss": 4.905914306640625,
+      "eval_runtime": 2.9951,
+      "eval_samples_per_second": 689.451,
+      "eval_steps_per_second": 43.404,
       "step": 22000
     },
     {
       "epoch": 30.0,
-      "grad_norm": 2.1233935356140137,
-      "learning_rate": 5.9980000000000005e-05,
-      "loss": 3.183,
+      "grad_norm": 5.983532428741455,
+      "learning_rate": 6e-05,
+      "loss": 4.6992,
       "step": 24000
     },
     {
       "epoch": 30.0,
-      "eval_loss": 3.421466112136841,
-      "eval_runtime": 2.3376,
-      "eval_samples_per_second": 883.393,
-      "eval_steps_per_second": 55.613,
+      "eval_loss": 4.167966842651367,
+      "eval_runtime": 2.4346,
+      "eval_samples_per_second": 848.205,
+      "eval_steps_per_second": 53.398,
       "step": 24000
     },
     {
       "epoch": 32.5,
-      "eval_loss": 3.3817927837371826,
-      "eval_runtime": 2.4614,
-      "eval_samples_per_second": 838.961,
-      "eval_steps_per_second": 52.816,
+      "eval_loss": 3.6409149169921875,
+      "eval_runtime": 2.8198,
+      "eval_samples_per_second": 732.313,
+      "eval_steps_per_second": 46.102,
       "step": 26000
     },
     {
       "epoch": 35.0,
-      "grad_norm": 2.176734209060669,
-      "learning_rate": 6.997500000000001e-05,
-      "loss": 3.0393,
+      "grad_norm": 4.979861259460449,
+      "learning_rate": 7e-05,
+      "loss": 3.5699,
       "step": 28000
     },
     {
       "epoch": 35.0,
-      "eval_loss": 3.34932541847229,
-      "eval_runtime": 2.3878,
-      "eval_samples_per_second": 864.811,
-      "eval_steps_per_second": 54.443,
+      "eval_loss": 3.2063941955566406,
+      "eval_runtime": 2.8452,
+      "eval_samples_per_second": 725.772,
+      "eval_steps_per_second": 45.69,
       "step": 28000
     },
     {
       "epoch": 37.5,
-      "eval_loss": 3.3345186710357666,
-      "eval_runtime": 2.3885,
-      "eval_samples_per_second": 864.564,
-      "eval_steps_per_second": 54.428,
+      "eval_loss": 3.0010480880737305,
+      "eval_runtime": 3.0576,
+      "eval_samples_per_second": 675.356,
+      "eval_steps_per_second": 42.516,
       "step": 30000
     },
     {
       "epoch": 40.0,
-      "grad_norm": 2.091628313064575,
-      "learning_rate": 7.997e-05,
-      "loss": 2.9118,
+      "grad_norm": 4.538495063781738,
+      "learning_rate": 8e-05,
+      "loss": 2.9074,
       "step": 32000
     },
     {
       "epoch": 40.0,
-      "eval_loss": 3.3137173652648926,
-      "eval_runtime": 2.3647,
-      "eval_samples_per_second": 873.264,
-      "eval_steps_per_second": 54.975,
+      "eval_loss": 2.850881338119507,
+      "eval_runtime": 3.0048,
+      "eval_samples_per_second": 687.232,
+      "eval_steps_per_second": 43.264,
       "step": 32000
     },
     {
       "epoch": 42.5,
-      "eval_loss": 3.3157804012298584,
-      "eval_runtime": 2.3512,
-      "eval_samples_per_second": 878.277,
-      "eval_steps_per_second": 55.291,
+      "eval_loss": 2.733860731124878,
+      "eval_runtime": 3.0203,
+      "eval_samples_per_second": 683.699,
+      "eval_steps_per_second": 43.042,
       "step": 34000
     },
     {
       "epoch": 45.0,
-      "grad_norm": 2.1319990158081055,
-      "learning_rate": 8.9965e-05,
-      "loss": 2.7946,
+      "grad_norm": 3.839367389678955,
+      "learning_rate": 9e-05,
+      "loss": 2.6073,
       "step": 36000
     },
     {
       "epoch": 45.0,
-      "eval_loss": 3.307403087615967,
-      "eval_runtime": 2.4041,
-      "eval_samples_per_second": 858.955,
-      "eval_steps_per_second": 54.075,
+      "eval_loss": 2.6182146072387695,
+      "eval_runtime": 3.0392,
+      "eval_samples_per_second": 679.464,
+      "eval_steps_per_second": 42.775,
       "step": 36000
     },
     {
       "epoch": 47.5,
-      "eval_loss": 3.3314850330352783,
-      "eval_runtime": 2.3564,
-      "eval_samples_per_second": 876.353,
-      "eval_steps_per_second": 55.17,
+      "eval_loss": 2.561305046081543,
+      "eval_runtime": 2.9856,
+      "eval_samples_per_second": 691.642,
+      "eval_steps_per_second": 43.542,
       "step": 38000
     },
     {
       "epoch": 50.0,
-      "grad_norm": 2.248058557510376,
-      "learning_rate": 9.996000000000001e-05,
-      "loss": 2.682,
+      "grad_norm": 5.377744674682617,
+      "learning_rate": 0.0001,
+      "loss": 2.4166,
       "step": 40000
     },
     {
       "epoch": 50.0,
-      "eval_loss": 3.3342697620391846,
-      "eval_runtime": 2.391,
-      "eval_samples_per_second": 863.666,
-      "eval_steps_per_second": 54.371,
+      "eval_loss": 2.4945614337921143,
+      "eval_runtime": 3.0785,
+      "eval_samples_per_second": 670.787,
+      "eval_steps_per_second": 42.229,
       "step": 40000
     },
     {
       "epoch": 52.5,
-      "eval_loss": 3.357624053955078,
-      "eval_runtime": 2.4114,
-      "eval_samples_per_second": 856.338,
-      "eval_steps_per_second": 53.91,
+      "eval_loss": 2.4196503162384033,
+      "eval_runtime": 3.1293,
+      "eval_samples_per_second": 659.901,
+      "eval_steps_per_second": 41.543,
       "step": 42000
     },
     {
       "epoch": 55.0,
-      "grad_norm": 2.307204008102417,
-      "learning_rate": 9.336333333333334e-05,
-      "loss": 2.5612,
+      "grad_norm": 4.3354268074035645,
+      "learning_rate": 9.333333333333334e-05,
+      "loss": 2.2667,
       "step": 44000
     },
     {
       "epoch": 55.0,
-      "eval_loss": 3.3740949630737305,
-      "eval_runtime": 2.3663,
-      "eval_samples_per_second": 872.673,
-      "eval_steps_per_second": 54.938,
+      "eval_loss": 2.3687477111816406,
+      "eval_runtime": 3.0418,
+      "eval_samples_per_second": 678.883,
+      "eval_steps_per_second": 42.738,
       "step": 44000
     },
     {
       "epoch": 57.5,
-      "eval_loss": 3.420975685119629,
-      "eval_runtime": 2.3864,
-      "eval_samples_per_second": 865.33,
-      "eval_steps_per_second": 54.476,
+      "eval_loss": 2.2801854610443115,
+      "eval_runtime": 2.562,
+      "eval_samples_per_second": 805.998,
+      "eval_steps_per_second": 50.741,
       "step": 46000
     },
     {
       "epoch": 60.0,
-      "grad_norm": 2.3412156105041504,
-      "learning_rate": 8.67e-05,
-      "loss": 2.4371,
+      "grad_norm": 3.7592601776123047,
+      "learning_rate": 8.666666666666667e-05,
+      "loss": 2.146,
       "step": 48000
     },
     {
       "epoch": 60.0,
-      "eval_loss": 3.4419877529144287,
-      "eval_runtime": 2.3487,
-      "eval_samples_per_second": 879.204,
-      "eval_steps_per_second": 55.349,
+      "eval_loss": 2.262115478515625,
+      "eval_runtime": 1.9284,
+      "eval_samples_per_second": 1070.809,
+      "eval_steps_per_second": 67.412,
       "step": 48000
     },
     {
       "epoch": 62.5,
-      "eval_loss": 3.492137908935547,
-      "eval_runtime": 2.3307,
-      "eval_samples_per_second": 886.014,
-      "eval_steps_per_second": 55.778,
+      "eval_loss": 2.2170372009277344,
+      "eval_runtime": 1.6322,
+      "eval_samples_per_second": 1265.182,
+      "eval_steps_per_second": 79.648,
       "step": 50000
     },
     {
       "epoch": 65.0,
-      "grad_norm": 2.591003179550171,
-      "learning_rate": 8.003666666666667e-05,
-      "loss": 2.3257,
+      "grad_norm": 4.673563480377197,
+      "learning_rate": 8e-05,
+      "loss": 2.0465,
       "step": 52000
     },
     {
       "epoch": 65.0,
-      "eval_loss": 3.511345624923706,
-      "eval_runtime": 2.3521,
-      "eval_samples_per_second": 877.926,
-      "eval_steps_per_second": 55.269,
+      "eval_loss": 2.1907379627227783,
+      "eval_runtime": 1.7,
+      "eval_samples_per_second": 1214.702,
+      "eval_steps_per_second": 76.47,
       "step": 52000
     },
     {
       "epoch": 67.5,
-      "eval_loss": 3.5672011375427246,
-      "eval_runtime": 2.3997,
-      "eval_samples_per_second": 860.512,
-      "eval_steps_per_second": 54.173,
+      "eval_loss": 2.1659305095672607,
+      "eval_runtime": 1.8055,
+      "eval_samples_per_second": 1143.712,
+      "eval_steps_per_second": 72.001,
       "step": 54000
     },
     {
       "epoch": 70.0,
-      "grad_norm": 2.841981887817383,
-      "learning_rate": 7.337333333333334e-05,
-      "loss": 2.2264,
+      "grad_norm": 3.939915180206299,
+      "learning_rate": 7.333333333333333e-05,
+      "loss": 1.969,
       "step": 56000
     },
     {
       "epoch": 70.0,
-      "eval_loss": 3.5860774517059326,
-      "eval_runtime": 2.3659,
-      "eval_samples_per_second": 872.827,
-      "eval_steps_per_second": 54.948,
+      "eval_loss": 2.127293825149536,
+      "eval_runtime": 1.7543,
+      "eval_samples_per_second": 1177.082,
+      "eval_steps_per_second": 74.102,
       "step": 56000
     },
     {
       "epoch": 72.5,
-      "eval_loss": 3.640416145324707,
-      "eval_runtime": 2.3679,
-      "eval_samples_per_second": 872.064,
-      "eval_steps_per_second": 54.9,
+      "eval_loss": 2.087448835372925,
+      "eval_runtime": 1.7243,
+      "eval_samples_per_second": 1197.568,
+      "eval_steps_per_second": 75.392,
       "step": 58000
     },
     {
       "epoch": 75.0,
-      "grad_norm": 3.0831592082977295,
-      "learning_rate": 6.671e-05,
-      "loss": 2.1379,
+      "grad_norm": 5.18148136138916,
+      "learning_rate": 6.666666666666667e-05,
+      "loss": 1.9056,
       "step": 60000
     },
     {
       "epoch": 75.0,
-      "eval_loss": 3.6711511611938477,
-      "eval_runtime": 2.357,
-      "eval_samples_per_second": 876.116,
-      "eval_steps_per_second": 55.155,
+      "eval_loss": 2.0742504596710205,
+      "eval_runtime": 1.7269,
+      "eval_samples_per_second": 1195.792,
+      "eval_steps_per_second": 75.28,
       "step": 60000
     },
     {
       "epoch": 77.5,
-      "eval_loss": 3.7177553176879883,
-      "eval_runtime": 2.3056,
-      "eval_samples_per_second": 895.662,
-      "eval_steps_per_second": 56.385,
+      "eval_loss": 2.0582902431488037,
+      "eval_runtime": 1.7674,
+      "eval_samples_per_second": 1168.411,
+      "eval_steps_per_second": 73.556,
       "step": 62000
     },
     {
       "epoch": 80.0,
-      "grad_norm": 3.1622650623321533,
-      "learning_rate": 6.0046666666666676e-05,
-      "loss": 2.0592,
+      "grad_norm": 4.553219795227051,
+      "learning_rate": 6e-05,
+      "loss": 1.8492,
       "step": 64000
     },
     {
       "epoch": 80.0,
-      "eval_loss": 3.730553388595581,
-      "eval_runtime": 2.365,
-      "eval_samples_per_second": 873.163,
-      "eval_steps_per_second": 54.969,
+      "eval_loss": 2.037095785140991,
+      "eval_runtime": 1.6339,
+      "eval_samples_per_second": 1263.872,
+      "eval_steps_per_second": 79.566,
       "step": 64000
     },
     {
       "epoch": 82.5,
-      "eval_loss": 3.7847235202789307,
-      "eval_runtime": 2.3591,
-      "eval_samples_per_second": 875.333,
-      "eval_steps_per_second": 55.106,
+      "eval_loss": 2.0038888454437256,
+      "eval_runtime": 1.6907,
+      "eval_samples_per_second": 1221.38,
+      "eval_steps_per_second": 76.891,
       "step": 66000
     },
     {
       "epoch": 85.0,
-      "grad_norm": 3.4362075328826904,
-      "learning_rate": 5.338333333333334e-05,
-      "loss": 1.9888,
+      "grad_norm": 3.77113938331604,
+      "learning_rate": 5.333333333333333e-05,
+      "loss": 1.8024,
       "step": 68000
     },
     {
       "epoch": 85.0,
-      "eval_loss": 3.8041844367980957,
-      "eval_runtime": 2.355,
-      "eval_samples_per_second": 876.853,
-      "eval_steps_per_second": 55.201,
+      "eval_loss": 1.9900726079940796,
+      "eval_runtime": 1.6879,
+      "eval_samples_per_second": 1223.403,
+      "eval_steps_per_second": 77.018,
       "step": 68000
     },
     {
       "epoch": 87.5,
-      "eval_loss": 3.858067750930786,
-      "eval_runtime": 2.3881,
-      "eval_samples_per_second": 864.721,
-      "eval_steps_per_second": 54.438,
+      "eval_loss": 1.9754005670547485,
+      "eval_runtime": 1.7314,
+      "eval_samples_per_second": 1192.693,
+      "eval_steps_per_second": 75.085,
       "step": 70000
     },
     {
       "epoch": 90.0,
-      "grad_norm": 3.4018688201904297,
-      "learning_rate": 4.672e-05,
-      "loss": 1.9259,
+      "grad_norm": 3.755357027053833,
+      "learning_rate": 4.666666666666667e-05,
+      "loss": 1.7652,
       "step": 72000
     },
     {
       "epoch": 90.0,
-      "eval_loss": 3.875187397003174,
-      "eval_runtime": 2.3891,
-      "eval_samples_per_second": 864.333,
-      "eval_steps_per_second": 54.413,
+      "eval_loss": 1.9565609693527222,
+      "eval_runtime": 1.8855,
+      "eval_samples_per_second": 1095.221,
+      "eval_steps_per_second": 68.949,
       "step": 72000
     },
     {
       "epoch": 92.5,
-      "eval_loss": 3.9157888889312744,
-      "eval_runtime": 2.3763,
-      "eval_samples_per_second": 869.013,
-      "eval_steps_per_second": 54.708,
+      "eval_loss": 1.940429449081421,
+      "eval_runtime": 1.7766,
+      "eval_samples_per_second": 1162.312,
+      "eval_steps_per_second": 73.172,
       "step": 74000
     },
     {
       "epoch": 95.0,
-      "grad_norm": 3.514496326446533,
-      "learning_rate": 4.005666666666667e-05,
-      "loss": 1.8696,
+      "grad_norm": 3.4141392707824707,
+      "learning_rate": 4e-05,
+      "loss": 1.7344,
       "step": 76000
     },
     {
       "epoch": 95.0,
-      "eval_loss": 3.928122043609619,
-      "eval_runtime": 2.3792,
-      "eval_samples_per_second": 867.948,
-      "eval_steps_per_second": 54.641,
+      "eval_loss": 1.9128267765045166,
+      "eval_runtime": 1.8509,
+      "eval_samples_per_second": 1115.674,
+      "eval_steps_per_second": 70.236,
       "step": 76000
     },
     {
       "epoch": 97.5,
-      "eval_loss": 3.9650816917419434,
-      "eval_runtime": 2.3646,
-      "eval_samples_per_second": 873.313,
-      "eval_steps_per_second": 54.979,
+      "eval_loss": 1.939635157585144,
+      "eval_runtime": 1.7798,
+      "eval_samples_per_second": 1160.26,
+      "eval_steps_per_second": 73.043,
       "step": 78000
     },
     {
       "epoch": 100.0,
-      "grad_norm": 3.6415059566497803,
-      "learning_rate": 3.339333333333334e-05,
-      "loss": 1.8186,
+      "grad_norm": 4.214694976806641,
+      "learning_rate": 3.3333333333333335e-05,
+      "loss": 1.7055,
       "step": 80000
     },
     {
       "epoch": 100.0,
-      "eval_loss": 3.984812021255493,
-      "eval_runtime": 2.3835,
-      "eval_samples_per_second": 866.363,
-      "eval_steps_per_second": 54.541,
+      "eval_loss": 1.959123134613037,
+      "eval_runtime": 1.6637,
+      "eval_samples_per_second": 1241.231,
+      "eval_steps_per_second": 78.14,
       "step": 80000
     },
     {
       "epoch": 102.5,
-      "eval_loss": 4.019808292388916,
-      "eval_runtime": 2.3534,
-      "eval_samples_per_second": 877.463,
-      "eval_steps_per_second": 55.24,
+      "eval_loss": 1.9078463315963745,
+      "eval_runtime": 1.6559,
+      "eval_samples_per_second": 1247.034,
+      "eval_steps_per_second": 78.506,
       "step": 82000
     },
     {
       "epoch": 105.0,
-      "grad_norm": 3.846658229827881,
-      "learning_rate": 2.673e-05,
-      "loss": 1.7735,
+      "grad_norm": 3.9869794845581055,
+      "learning_rate": 2.6666666666666667e-05,
+      "loss": 1.6779,
       "step": 84000
     },
     {
       "epoch": 105.0,
-      "eval_loss": 4.02935266494751,
-      "eval_runtime": 2.3869,
-      "eval_samples_per_second": 865.136,
-      "eval_steps_per_second": 54.464,
+      "eval_loss": 1.9178215265274048,
+      "eval_runtime": 1.7131,
+      "eval_samples_per_second": 1205.385,
+      "eval_steps_per_second": 75.884,
       "step": 84000
     },
     {
       "epoch": 107.5,
-      "eval_loss": 4.060792446136475,
-      "eval_runtime": 2.4253,
-      "eval_samples_per_second": 851.431,
-      "eval_steps_per_second": 53.601,
+      "eval_loss": 1.9046084880828857,
+      "eval_runtime": 1.7051,
+      "eval_samples_per_second": 1211.05,
+      "eval_steps_per_second": 76.24,
       "step": 86000
     },
     {
       "epoch": 110.0,
-      "grad_norm": 3.832317352294922,
-      "learning_rate": 2.0066666666666665e-05,
-      "loss": 1.7319,
+      "grad_norm": 3.5100035667419434,
+      "learning_rate": 2e-05,
+      "loss": 1.6529,
       "step": 88000
     },
     {
       "epoch": 110.0,
-      "eval_loss": 4.0724196434021,
-      "eval_runtime": 2.3704,
-      "eval_samples_per_second": 871.154,
-      "eval_steps_per_second": 54.843,
+      "eval_loss": 1.8918195962905884,
+      "eval_runtime": 1.7296,
+      "eval_samples_per_second": 1193.906,
+      "eval_steps_per_second": 75.161,
       "step": 88000
     },
     {
       "epoch": 112.5,
-      "eval_loss": 4.093896865844727,
-      "eval_runtime": 2.3692,
-      "eval_samples_per_second": 871.585,
-      "eval_steps_per_second": 54.87,
+      "eval_loss": 1.9009567499160767,
+      "eval_runtime": 1.6723,
+      "eval_samples_per_second": 1234.829,
+      "eval_steps_per_second": 77.737,
       "step": 90000
     },
     {
       "epoch": 115.0,
-      "grad_norm": 3.938507556915283,
-      "learning_rate": 1.3403333333333334e-05,
-      "loss": 1.6956,
+      "grad_norm": 5.523593425750732,
+      "learning_rate": 1.3333333333333333e-05,
+      "loss": 1.6347,
       "step": 92000
     },
     {
       "epoch": 115.0,
-      "eval_loss": 4.105323314666748,
-      "eval_runtime": 2.3654,
-      "eval_samples_per_second": 872.995,
-      "eval_steps_per_second": 54.959,
+      "eval_loss": 1.8959263563156128,
+      "eval_runtime": 2.9066,
+      "eval_samples_per_second": 710.446,
+      "eval_steps_per_second": 44.725,
       "step": 92000
     },
     {
       "epoch": 117.5,
-      "eval_loss": 4.126801013946533,
-      "eval_runtime": 2.3732,
-      "eval_samples_per_second": 870.126,
-      "eval_steps_per_second": 54.778,
+      "eval_loss": 1.909432291984558,
+      "eval_runtime": 3.2031,
+      "eval_samples_per_second": 644.687,
+      "eval_steps_per_second": 40.586,
       "step": 94000
     },
     {
       "epoch": 120.0,
-      "grad_norm": 3.9486958980560303,
-      "learning_rate": 6.740000000000001e-06,
-      "loss": 1.6642,
+      "grad_norm": 2.9994258880615234,
+      "learning_rate": 6.666666666666667e-06,
+      "loss": 1.6225,
       "step": 96000
     },
     {
       "epoch": 120.0,
-      "eval_loss": 4.135463714599609,
-      "eval_runtime": 2.394,
-      "eval_samples_per_second": 862.577,
-      "eval_steps_per_second": 54.303,
+      "eval_loss": 1.8838213682174683,
+      "eval_runtime": 3.0544,
+      "eval_samples_per_second": 676.065,
+      "eval_steps_per_second": 42.561,
       "step": 96000
     }
   ],
@@ -578,7 +578,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2.4822358813999104e+16,
+  "total_flos": 2.9945921273856e+16,
   "train_batch_size": 16,
   "trial_name": null,
   "trial_params": null

checkpoint-96000/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:eb433b87752cc469b20ee625a833c904bc80363d908ad5bc61687e872bf3a42a
+oid sha256:b7e62d267b0e2dac82b0654ee993033d8fc2441a7f6a20ed53b7f7b3af048d8d
 size 5368