Upload folder using huggingface_hub

checkpoint-12208/config.json

@@ -0,0 +1,39 @@
+{
+  "_name_or_path": "meta-llama/Llama-3.1-8B-Instruct",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.0.dev0",
+  "use_cache": true,
+  "vocab_size": 128256
+}

checkpoint-12208/generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.44.0.dev0"
+}

checkpoint-12208/model-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6e41dc86657c1d2149738fb30989a534d038c0963531fbe129454b5636019e9
+size 4886466168

checkpoint-12208/model-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23fa5aaf8ac56c17c821980c9aee44fcbc2a617fc570e875e6fdb93d92886c72
+size 4832007448

checkpoint-12208/model-00003-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a223e87240e50dfb05634ea909cf1ec715d37a12b681b14a3a6e97f28688f2b
+size 4999813112

checkpoint-12208/model-00004-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3047d199c611c6d12b1cca780f033ad89df9c076f742674f7ac4a892ca9573b3
+size 4999813128

checkpoint-12208/model-00005-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a2f91d749d664ecd22c71f3d54488966fc6899a03924b8fea3613c8f83398a8
+size 4832007496

checkpoint-12208/model-00006-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:408c757804e15f394790e0cfec6017deb32b82842faab9eecf7ab9bcc32438c4
+size 4999813120

checkpoint-12208/model-00007-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ee6209ba80ffef498bffbf5f595cd96fb8285b879825313838ee31452ff8a19
+size 2571158184

checkpoint-12208/model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 32121044992
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00007-of-00007.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.norm.weight": "model-00007-of-00007.safetensors"
+  }
+}

checkpoint-12208/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54483e5c3c8e397bc48b5f28fd24bcdc890e169b837ca1fe9b352ca2db48f80c
+size 15385036334

checkpoint-12208/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
+size 14244

checkpoint-12208/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:abe644ed33a3c4139223f0857a985127f3e6fbaa8c89fa14b57671b49ca52c21
+size 1064

checkpoint-12208/trainer_state.json

@@ -0,0 +1,2784 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.08560019352599454,
+  "eval_steps": 500,
+  "global_step": 12208,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.00021736615328520894,
+      "grad_norm": 5.709163665771484,
+      "learning_rate": 1.0157273918741808e-06,
+      "loss": 0.9228,
+      "step": 31
+    },
+    {
+      "epoch": 0.0004347323065704179,
+      "grad_norm": 3.9758756160736084,
+      "learning_rate": 2.0314547837483616e-06,
+      "loss": 0.8276,
+      "step": 62
+    },
+    {
+      "epoch": 0.0006520984598556268,
+      "grad_norm": 3.721677303314209,
+      "learning_rate": 3.0471821756225426e-06,
+      "loss": 0.7897,
+      "step": 93
+    },
+    {
+      "epoch": 0.0008694646131408358,
+      "grad_norm": 3.384953498840332,
+      "learning_rate": 4.062909567496723e-06,
+      "loss": 0.7641,
+      "step": 124
+    },
+    {
+      "epoch": 0.0010868307664260446,
+      "grad_norm": 3.19750714302063,
+      "learning_rate": 5.078636959370905e-06,
+      "loss": 0.758,
+      "step": 155
+    },
+    {
+      "epoch": 0.0013041969197112536,
+      "grad_norm": 3.49003267288208,
+      "learning_rate": 6.094364351245085e-06,
+      "loss": 0.7644,
+      "step": 186
+    },
+    {
+      "epoch": 0.0015215630729964625,
+      "grad_norm": 3.323367118835449,
+      "learning_rate": 7.110091743119267e-06,
+      "loss": 0.7606,
+      "step": 217
+    },
+    {
+      "epoch": 0.0017389292262816715,
+      "grad_norm": 3.465822219848633,
+      "learning_rate": 8.125819134993446e-06,
+      "loss": 0.7505,
+      "step": 248
+    },
+    {
+      "epoch": 0.0019562953795668804,
+      "grad_norm": 2.7967450618743896,
+      "learning_rate": 9.141546526867629e-06,
+      "loss": 0.7593,
+      "step": 279
+    },
+    {
+      "epoch": 0.002173661532852089,
+      "grad_norm": 3.5493738651275635,
+      "learning_rate": 1.015727391874181e-05,
+      "loss": 0.7514,
+      "step": 310
+    },
+    {
+      "epoch": 0.0023910276861372984,
+      "grad_norm": 3.514606237411499,
+      "learning_rate": 1.117300131061599e-05,
+      "loss": 0.7497,
+      "step": 341
+    },
+    {
+      "epoch": 0.0026083938394225073,
+      "grad_norm": 3.0246028900146484,
+      "learning_rate": 1.218872870249017e-05,
+      "loss": 0.7473,
+      "step": 372
+    },
+    {
+      "epoch": 0.002825759992707716,
+      "grad_norm": 3.01147723197937,
+      "learning_rate": 1.3204456094364351e-05,
+      "loss": 0.739,
+      "step": 403
+    },
+    {
+      "epoch": 0.003043126145992925,
+      "grad_norm": 3.566333293914795,
+      "learning_rate": 1.4220183486238533e-05,
+      "loss": 0.7447,
+      "step": 434
+    },
+    {
+      "epoch": 0.003260492299278134,
+      "grad_norm": 3.8698947429656982,
+      "learning_rate": 1.5235910878112714e-05,
+      "loss": 0.7447,
+      "step": 465
+    },
+    {
+      "epoch": 0.003477858452563343,
+      "grad_norm": 2.567028760910034,
+      "learning_rate": 1.6251638269986893e-05,
+      "loss": 0.7496,
+      "step": 496
+    },
+    {
+      "epoch": 0.003695224605848552,
+      "grad_norm": 2.8029377460479736,
+      "learning_rate": 1.7267365661861077e-05,
+      "loss": 0.7324,
+      "step": 527
+    },
+    {
+      "epoch": 0.003912590759133761,
+      "grad_norm": 2.862530469894409,
+      "learning_rate": 1.8283093053735257e-05,
+      "loss": 0.7371,
+      "step": 558
+    },
+    {
+      "epoch": 0.00412995691241897,
+      "grad_norm": 2.9063901901245117,
+      "learning_rate": 1.9298820445609438e-05,
+      "loss": 0.7398,
+      "step": 589
+    },
+    {
+      "epoch": 0.004347323065704178,
+      "grad_norm": 3.187814235687256,
+      "learning_rate": 2.031454783748362e-05,
+      "loss": 0.7362,
+      "step": 620
+    },
+    {
+      "epoch": 0.004564689218989388,
+      "grad_norm": 2.6158969402313232,
+      "learning_rate": 2.13302752293578e-05,
+      "loss": 0.7405,
+      "step": 651
+    },
+    {
+      "epoch": 0.004782055372274597,
+      "grad_norm": 2.702423334121704,
+      "learning_rate": 2.234600262123198e-05,
+      "loss": 0.7313,
+      "step": 682
+    },
+    {
+      "epoch": 0.004999421525559805,
+      "grad_norm": 2.88313889503479,
+      "learning_rate": 2.336173001310616e-05,
+      "loss": 0.7337,
+      "step": 713
+    },
+    {
+      "epoch": 0.0052167876788450146,
+      "grad_norm": 2.8978841304779053,
+      "learning_rate": 2.437745740498034e-05,
+      "loss": 0.73,
+      "step": 744
+    },
+    {
+      "epoch": 0.005434153832130223,
+      "grad_norm": 2.791414976119995,
+      "learning_rate": 2.5393184796854525e-05,
+      "loss": 0.7297,
+      "step": 775
+    },
+    {
+      "epoch": 0.005651519985415432,
+      "grad_norm": 2.432821035385132,
+      "learning_rate": 2.6408912188728702e-05,
+      "loss": 0.7326,
+      "step": 806
+    },
+    {
+      "epoch": 0.0058688861387006415,
+      "grad_norm": 2.430279493331909,
+      "learning_rate": 2.7424639580602886e-05,
+      "loss": 0.7333,
+      "step": 837
+    },
+    {
+      "epoch": 0.00608625229198585,
+      "grad_norm": 2.5513761043548584,
+      "learning_rate": 2.8440366972477066e-05,
+      "loss": 0.7296,
+      "step": 868
+    },
+    {
+      "epoch": 0.006303618445271059,
+      "grad_norm": 2.478562116622925,
+      "learning_rate": 2.9456094364351244e-05,
+      "loss": 0.7303,
+      "step": 899
+    },
+    {
+      "epoch": 0.006520984598556268,
+      "grad_norm": 2.5496723651885986,
+      "learning_rate": 3.0471821756225428e-05,
+      "loss": 0.7303,
+      "step": 930
+    },
+    {
+      "epoch": 0.006738350751841477,
+      "grad_norm": 2.114060640335083,
+      "learning_rate": 3.148754914809961e-05,
+      "loss": 0.7184,
+      "step": 961
+    },
+    {
+      "epoch": 0.006955716905126686,
+      "grad_norm": 2.325977325439453,
+      "learning_rate": 3.2503276539973785e-05,
+      "loss": 0.7268,
+      "step": 992
+    },
+    {
+      "epoch": 0.0071730830584118945,
+      "grad_norm": 2.0549662113189697,
+      "learning_rate": 3.351900393184797e-05,
+      "loss": 0.724,
+      "step": 1023
+    },
+    {
+      "epoch": 0.007390449211697104,
+      "grad_norm": 1.913522481918335,
+      "learning_rate": 3.453473132372215e-05,
+      "loss": 0.7206,
+      "step": 1054
+    },
+    {
+      "epoch": 0.007607815364982313,
+      "grad_norm": 2.0651443004608154,
+      "learning_rate": 3.555045871559633e-05,
+      "loss": 0.7239,
+      "step": 1085
+    },
+    {
+      "epoch": 0.007825181518267521,
+      "grad_norm": 2.2482309341430664,
+      "learning_rate": 3.6566186107470514e-05,
+      "loss": 0.728,
+      "step": 1116
+    },
+    {
+      "epoch": 0.00804254767155273,
+      "grad_norm": 2.349695920944214,
+      "learning_rate": 3.7581913499344695e-05,
+      "loss": 0.721,
+      "step": 1147
+    },
+    {
+      "epoch": 0.00825991382483794,
+      "grad_norm": 2.218843460083008,
+      "learning_rate": 3.8597640891218876e-05,
+      "loss": 0.7248,
+      "step": 1178
+    },
+    {
+      "epoch": 0.00847727997812315,
+      "grad_norm": 2.0839340686798096,
+      "learning_rate": 3.9613368283093056e-05,
+      "loss": 0.7233,
+      "step": 1209
+    },
+    {
+      "epoch": 0.008694646131408357,
+      "grad_norm": 2.0757343769073486,
+      "learning_rate": 4.062909567496724e-05,
+      "loss": 0.7166,
+      "step": 1240
+    },
+    {
+      "epoch": 0.008912012284693566,
+      "grad_norm": 2.052342653274536,
+      "learning_rate": 4.164482306684142e-05,
+      "loss": 0.7256,
+      "step": 1271
+    },
+    {
+      "epoch": 0.009129378437978775,
+      "grad_norm": 2.1202704906463623,
+      "learning_rate": 4.26605504587156e-05,
+      "loss": 0.7131,
+      "step": 1302
+    },
+    {
+      "epoch": 0.009346744591263985,
+      "grad_norm": 2.1945958137512207,
+      "learning_rate": 4.367627785058978e-05,
+      "loss": 0.708,
+      "step": 1333
+    },
+    {
+      "epoch": 0.009564110744549194,
+      "grad_norm": 2.106307029724121,
+      "learning_rate": 4.469200524246396e-05,
+      "loss": 0.7139,
+      "step": 1364
+    },
+    {
+      "epoch": 0.009781476897834401,
+      "grad_norm": 2.3779594898223877,
+      "learning_rate": 4.570773263433814e-05,
+      "loss": 0.7205,
+      "step": 1395
+    },
+    {
+      "epoch": 0.00999884305111961,
+      "grad_norm": 2.001551866531372,
+      "learning_rate": 4.672346002621232e-05,
+      "loss": 0.7162,
+      "step": 1426
+    },
+    {
+      "epoch": 0.01021620920440482,
+      "grad_norm": 2.6994752883911133,
+      "learning_rate": 4.77391874180865e-05,
+      "loss": 0.7215,
+      "step": 1457
+    },
+    {
+      "epoch": 0.010433575357690029,
+      "grad_norm": 2.071122884750366,
+      "learning_rate": 4.875491480996068e-05,
+      "loss": 0.7095,
+      "step": 1488
+    },
+    {
+      "epoch": 0.010650941510975238,
+      "grad_norm": 2.3666610717773438,
+      "learning_rate": 4.977064220183487e-05,
+      "loss": 0.7089,
+      "step": 1519
+    },
+    {
+      "epoch": 0.010868307664260446,
+      "grad_norm": 2.130204677581787,
+      "learning_rate": 4.9999915451558777e-05,
+      "loss": 0.7043,
+      "step": 1550
+    },
+    {
+      "epoch": 0.011085673817545655,
+      "grad_norm": 1.7413716316223145,
+      "learning_rate": 4.999955597496219e-05,
+      "loss": 0.7062,
+      "step": 1581
+    },
+    {
+      "epoch": 0.011303039970830864,
+      "grad_norm": 1.7087843418121338,
+      "learning_rate": 4.9998914381774255e-05,
+      "loss": 0.7175,
+      "step": 1612
+    },
+    {
+      "epoch": 0.011520406124116074,
+      "grad_norm": 2.574871301651001,
+      "learning_rate": 4.999799067923527e-05,
+      "loss": 0.7076,
+      "step": 1643
+    },
+    {
+      "epoch": 0.011737772277401283,
+      "grad_norm": 2.0530433654785156,
+      "learning_rate": 4.999678487776908e-05,
+      "loss": 0.708,
+      "step": 1674
+    },
+    {
+      "epoch": 0.011955138430686492,
+      "grad_norm": 1.9184463024139404,
+      "learning_rate": 4.9995296990983006e-05,
+      "loss": 0.7011,
+      "step": 1705
+    },
+    {
+      "epoch": 0.0121725045839717,
+      "grad_norm": 1.768115520477295,
+      "learning_rate": 4.999352703566763e-05,
+      "loss": 0.6995,
+      "step": 1736
+    },
+    {
+      "epoch": 0.012389870737256909,
+      "grad_norm": 1.6692063808441162,
+      "learning_rate": 4.999147503179668e-05,
+      "loss": 0.7019,
+      "step": 1767
+    },
+    {
+      "epoch": 0.012607236890542118,
+      "grad_norm": 1.803944706916809,
+      "learning_rate": 4.998914100252672e-05,
+      "loss": 0.703,
+      "step": 1798
+    },
+    {
+      "epoch": 0.012824603043827328,
+      "grad_norm": 1.8957322835922241,
+      "learning_rate": 4.998652497419696e-05,
+      "loss": 0.6979,
+      "step": 1829
+    },
+    {
+      "epoch": 0.013041969197112537,
+      "grad_norm": 1.544054627418518,
+      "learning_rate": 4.9983626976328927e-05,
+      "loss": 0.7012,
+      "step": 1860
+    },
+    {
+      "epoch": 0.013259335350397744,
+      "grad_norm": 1.7066351175308228,
+      "learning_rate": 4.998044704162613e-05,
+      "loss": 0.6949,
+      "step": 1891
+    },
+    {
+      "epoch": 0.013476701503682954,
+      "grad_norm": 1.7045214176177979,
+      "learning_rate": 4.9976985205973705e-05,
+      "loss": 0.6871,
+      "step": 1922
+    },
+    {
+      "epoch": 0.013694067656968163,
+      "grad_norm": 2.0414485931396484,
+      "learning_rate": 4.997324150843799e-05,
+      "loss": 0.6944,
+      "step": 1953
+    },
+    {
+      "epoch": 0.013911433810253372,
+      "grad_norm": 1.9210485219955444,
+      "learning_rate": 4.99692159912661e-05,
+      "loss": 0.7013,
+      "step": 1984
+    },
+    {
+      "epoch": 0.014128799963538581,
+      "grad_norm": 1.8733997344970703,
+      "learning_rate": 4.996490869988546e-05,
+      "loss": 0.6991,
+      "step": 2015
+    },
+    {
+      "epoch": 0.014346166116823789,
+      "grad_norm": 1.7118934392929077,
+      "learning_rate": 4.996031968290326e-05,
+      "loss": 0.6873,
+      "step": 2046
+    },
+    {
+      "epoch": 0.014563532270108998,
+      "grad_norm": 1.889208197593689,
+      "learning_rate": 4.995544899210594e-05,
+      "loss": 0.692,
+      "step": 2077
+    },
+    {
+      "epoch": 0.014780898423394207,
+      "grad_norm": 2.0081522464752197,
+      "learning_rate": 4.9950296682458583e-05,
+      "loss": 0.6823,
+      "step": 2108
+    },
+    {
+      "epoch": 0.014998264576679417,
+      "grad_norm": 1.7950328588485718,
+      "learning_rate": 4.994486281210429e-05,
+      "loss": 0.686,
+      "step": 2139
+    },
+    {
+      "epoch": 0.015215630729964626,
+      "grad_norm": 1.758333444595337,
+      "learning_rate": 4.9939147442363566e-05,
+      "loss": 0.6827,
+      "step": 2170
+    },
+    {
+      "epoch": 0.015432996883249834,
+      "grad_norm": 1.496063470840454,
+      "learning_rate": 4.9933150637733574e-05,
+      "loss": 0.6896,
+      "step": 2201
+    },
+    {
+      "epoch": 0.015650363036535043,
+      "grad_norm": 1.6103190183639526,
+      "learning_rate": 4.992687246588743e-05,
+      "loss": 0.6865,
+      "step": 2232
+    },
+    {
+      "epoch": 0.015867729189820254,
+      "grad_norm": 1.7069604396820068,
+      "learning_rate": 4.992031299767347e-05,
+      "loss": 0.6836,
+      "step": 2263
+    },
+    {
+      "epoch": 0.01608509534310546,
+      "grad_norm": 1.8261148929595947,
+      "learning_rate": 4.9913472307114386e-05,
+      "loss": 0.692,
+      "step": 2294
+    },
+    {
+      "epoch": 0.01630246149639067,
+      "grad_norm": 1.7522642612457275,
+      "learning_rate": 4.9906350471406446e-05,
+      "loss": 0.6832,
+      "step": 2325
+    },
+    {
+      "epoch": 0.01651982764967588,
+      "grad_norm": 1.6746042966842651,
+      "learning_rate": 4.989894757091861e-05,
+      "loss": 0.6829,
+      "step": 2356
+    },
+    {
+      "epoch": 0.016737193802961087,
+      "grad_norm": 1.6785473823547363,
+      "learning_rate": 4.989126368919158e-05,
+      "loss": 0.6842,
+      "step": 2387
+    },
+    {
+      "epoch": 0.0169545599562463,
+      "grad_norm": 1.936916470527649,
+      "learning_rate": 4.988329891293693e-05,
+      "loss": 0.6798,
+      "step": 2418
+    },
+    {
+      "epoch": 0.017171926109531506,
+      "grad_norm": 1.5104129314422607,
+      "learning_rate": 4.987505333203608e-05,
+      "loss": 0.6743,
+      "step": 2449
+    },
+    {
+      "epoch": 0.017389292262816713,
+      "grad_norm": 1.3770678043365479,
+      "learning_rate": 4.9866527039539276e-05,
+      "loss": 0.6874,
+      "step": 2480
+    },
+    {
+      "epoch": 0.017606658416101924,
+      "grad_norm": 1.4900861978530884,
+      "learning_rate": 4.9857720131664594e-05,
+      "loss": 0.679,
+      "step": 2511
+    },
+    {
+      "epoch": 0.017824024569387132,
+      "grad_norm": 1.5600172281265259,
+      "learning_rate": 4.9848632707796773e-05,
+      "loss": 0.6719,
+      "step": 2542
+    },
+    {
+      "epoch": 0.018041390722672343,
+      "grad_norm": 1.660237431526184,
+      "learning_rate": 4.9839264870486155e-05,
+      "loss": 0.6773,
+      "step": 2573
+    },
+    {
+      "epoch": 0.01825875687595755,
+      "grad_norm": 1.4618791341781616,
+      "learning_rate": 4.9829616725447526e-05,
+      "loss": 0.6767,
+      "step": 2604
+    },
+    {
+      "epoch": 0.018476123029242758,
+      "grad_norm": 1.5567563772201538,
+      "learning_rate": 4.981968838155888e-05,
+      "loss": 0.6706,
+      "step": 2635
+    },
+    {
+      "epoch": 0.01869348918252797,
+      "grad_norm": 1.4879790544509888,
+      "learning_rate": 4.980947995086024e-05,
+      "loss": 0.676,
+      "step": 2666
+    },
+    {
+      "epoch": 0.018910855335813177,
+      "grad_norm": 1.5527766942977905,
+      "learning_rate": 4.979899154855234e-05,
+      "loss": 0.6734,
+      "step": 2697
+    },
+    {
+      "epoch": 0.019128221489098388,
+      "grad_norm": 1.7922642230987549,
+      "learning_rate": 4.9788223292995386e-05,
+      "loss": 0.6813,
+      "step": 2728
+    },
+    {
+      "epoch": 0.019345587642383595,
+      "grad_norm": 1.6453475952148438,
+      "learning_rate": 4.977717530570768e-05,
+      "loss": 0.6749,
+      "step": 2759
+    },
+    {
+      "epoch": 0.019562953795668803,
+      "grad_norm": 1.4713648557662964,
+      "learning_rate": 4.976584771136425e-05,
+      "loss": 0.6761,
+      "step": 2790
+    },
+    {
+      "epoch": 0.019780319948954014,
+      "grad_norm": 1.5513399839401245,
+      "learning_rate": 4.975424063779547e-05,
+      "loss": 0.6739,
+      "step": 2821
+    },
+    {
+      "epoch": 0.01999768610223922,
+      "grad_norm": 1.431796908378601,
+      "learning_rate": 4.974235421598557e-05,
+      "loss": 0.6639,
+      "step": 2852
+    },
+    {
+      "epoch": 0.020215052255524432,
+      "grad_norm": 1.4959752559661865,
+      "learning_rate": 4.973018858007122e-05,
+      "loss": 0.662,
+      "step": 2883
+    },
+    {
+      "epoch": 0.02043241840880964,
+      "grad_norm": 1.4675205945968628,
+      "learning_rate": 4.9717743867339963e-05,
+      "loss": 0.6659,
+      "step": 2914
+    },
+    {
+      "epoch": 0.020649784562094847,
+      "grad_norm": 1.7111692428588867,
+      "learning_rate": 4.9705020218228695e-05,
+      "loss": 0.6732,
+      "step": 2945
+    },
+    {
+      "epoch": 0.020867150715380058,
+      "grad_norm": 1.542748212814331,
+      "learning_rate": 4.969201777632205e-05,
+      "loss": 0.6666,
+      "step": 2976
+    },
+    {
+      "epoch": 0.021084516868665266,
+      "grad_norm": 1.5013272762298584,
+      "learning_rate": 4.9678736688350846e-05,
+      "loss": 0.667,
+      "step": 3007
+    },
+    {
+      "epoch": 0.021301883021950477,
+      "grad_norm": 1.57589852809906,
+      "learning_rate": 4.966517710419033e-05,
+      "loss": 0.6668,
+      "step": 3038
+    },
+    {
+      "epoch": 0.021519249175235684,
+      "grad_norm": 1.612727403640747,
+      "learning_rate": 4.965133917685858e-05,
+      "loss": 0.6653,
+      "step": 3069
+    },
+    {
+      "epoch": 0.021736615328520892,
+      "grad_norm": 1.5110771656036377,
+      "learning_rate": 4.9637223062514714e-05,
+      "loss": 0.6694,
+      "step": 3100
+    },
+    {
+      "epoch": 0.021953981481806103,
+      "grad_norm": 1.5836044549942017,
+      "learning_rate": 4.962282892045718e-05,
+      "loss": 0.6634,
+      "step": 3131
+    },
+    {
+      "epoch": 0.02217134763509131,
+      "grad_norm": 1.5767654180526733,
+      "learning_rate": 4.9608156913121904e-05,
+      "loss": 0.6633,
+      "step": 3162
+    },
+    {
+      "epoch": 0.02238871378837652,
+      "grad_norm": 1.6058984994888306,
+      "learning_rate": 4.959320720608049e-05,
+      "loss": 0.6646,
+      "step": 3193
+    },
+    {
+      "epoch": 0.02260607994166173,
+      "grad_norm": 1.4564005136489868,
+      "learning_rate": 4.9577979968038354e-05,
+      "loss": 0.663,
+      "step": 3224
+    },
+    {
+      "epoch": 0.022823446094946936,
+      "grad_norm": 1.5274450778961182,
+      "learning_rate": 4.956247537083282e-05,
+      "loss": 0.6554,
+      "step": 3255
+    },
+    {
+      "epoch": 0.023040812248232147,
+      "grad_norm": 1.524122953414917,
+      "learning_rate": 4.9546693589431145e-05,
+      "loss": 0.6666,
+      "step": 3286
+    },
+    {
+      "epoch": 0.023258178401517355,
+      "grad_norm": 1.5017430782318115,
+      "learning_rate": 4.9530634801928595e-05,
+      "loss": 0.6661,
+      "step": 3317
+    },
+    {
+      "epoch": 0.023475544554802566,
+      "grad_norm": 1.4247208833694458,
+      "learning_rate": 4.9514299189546395e-05,
+      "loss": 0.6563,
+      "step": 3348
+    },
+    {
+      "epoch": 0.023692910708087773,
+      "grad_norm": 1.3451093435287476,
+      "learning_rate": 4.949768693662973e-05,
+      "loss": 0.6555,
+      "step": 3379
+    },
+    {
+      "epoch": 0.023910276861372984,
+      "grad_norm": 1.418442726135254,
+      "learning_rate": 4.948079823064559e-05,
+      "loss": 0.6587,
+      "step": 3410
+    },
+    {
+      "epoch": 0.024127643014658192,
+      "grad_norm": 1.5460575819015503,
+      "learning_rate": 4.946363326218074e-05,
+      "loss": 0.6602,
+      "step": 3441
+    },
+    {
+      "epoch": 0.0243450091679434,
+      "grad_norm": 1.4053966999053955,
+      "learning_rate": 4.9446192224939525e-05,
+      "loss": 0.6578,
+      "step": 3472
+    },
+    {
+      "epoch": 0.02456237532122861,
+      "grad_norm": 1.420806646347046,
+      "learning_rate": 4.942847531574167e-05,
+      "loss": 0.6544,
+      "step": 3503
+    },
+    {
+      "epoch": 0.024779741474513818,
+      "grad_norm": 1.3220508098602295,
+      "learning_rate": 4.941048273452008e-05,
+      "loss": 0.6642,
+      "step": 3534
+    },
+    {
+      "epoch": 0.02499710762779903,
+      "grad_norm": 1.3819468021392822,
+      "learning_rate": 4.9392214684318605e-05,
+      "loss": 0.6534,
+      "step": 3565
+    },
+    {
+      "epoch": 0.025214473781084237,
+      "grad_norm": 1.4046236276626587,
+      "learning_rate": 4.93736713712897e-05,
+      "loss": 0.6547,
+      "step": 3596
+    },
+    {
+      "epoch": 0.025431839934369444,
+      "grad_norm": 1.965153455734253,
+      "learning_rate": 4.9354853004692124e-05,
+      "loss": 0.6582,
+      "step": 3627
+    },
+    {
+      "epoch": 0.025649206087654655,
+      "grad_norm": 1.6758291721343994,
+      "learning_rate": 4.93357597968886e-05,
+      "loss": 0.6536,
+      "step": 3658
+    },
+    {
+      "epoch": 0.025866572240939863,
+      "grad_norm": 1.4467246532440186,
+      "learning_rate": 4.931639196334338e-05,
+      "loss": 0.6564,
+      "step": 3689
+    },
+    {
+      "epoch": 0.026083938394225074,
+      "grad_norm": 1.5305490493774414,
+      "learning_rate": 4.9296749722619826e-05,
+      "loss": 0.6626,
+      "step": 3720
+    },
+    {
+      "epoch": 0.02630130454751028,
+      "grad_norm": 1.3826133012771606,
+      "learning_rate": 4.9276833296377966e-05,
+      "loss": 0.6624,
+      "step": 3751
+    },
+    {
+      "epoch": 0.02651867070079549,
+      "grad_norm": 1.4920278787612915,
+      "learning_rate": 4.925664290937196e-05,
+      "loss": 0.6417,
+      "step": 3782
+    },
+    {
+      "epoch": 0.0267360368540807,
+      "grad_norm": 14.533783912658691,
+      "learning_rate": 4.9236178789447576e-05,
+      "loss": 0.647,
+      "step": 3813
+    },
+    {
+      "epoch": 0.026953403007365907,
+      "grad_norm": 1.4659409523010254,
+      "learning_rate": 4.921544116753962e-05,
+      "loss": 0.6514,
+      "step": 3844
+    },
+    {
+      "epoch": 0.02717076916065112,
+      "grad_norm": 1.4820138216018677,
+      "learning_rate": 4.919443027766935e-05,
+      "loss": 0.6477,
+      "step": 3875
+    },
+    {
+      "epoch": 0.027388135313936326,
+      "grad_norm": 1.2543880939483643,
+      "learning_rate": 4.91731463569418e-05,
+      "loss": 0.6505,
+      "step": 3906
+    },
+    {
+      "epoch": 0.027605501467221533,
+      "grad_norm": 1.640926718711853,
+      "learning_rate": 4.915158964554312e-05,
+      "loss": 0.6488,
+      "step": 3937
+    },
+    {
+      "epoch": 0.027822867620506744,
+      "grad_norm": 1.3990615606307983,
+      "learning_rate": 4.912976038673786e-05,
+      "loss": 0.658,
+      "step": 3968
+    },
+    {
+      "epoch": 0.028040233773791952,
+      "grad_norm": 1.2064067125320435,
+      "learning_rate": 4.9107658826866254e-05,
+      "loss": 0.6487,
+      "step": 3999
+    },
+    {
+      "epoch": 0.028257599927077163,
+      "grad_norm": 1.5103769302368164,
+      "learning_rate": 4.908528521534139e-05,
+      "loss": 0.6569,
+      "step": 4030
+    },
+    {
+      "epoch": 0.02847496608036237,
+      "grad_norm": 1.5314087867736816,
+      "learning_rate": 4.906263980464644e-05,
+      "loss": 0.6638,
+      "step": 4061
+    },
+    {
+      "epoch": 0.028692332233647578,
+      "grad_norm": 1.4619168043136597,
+      "learning_rate": 4.903972285033178e-05,
+      "loss": 0.6505,
+      "step": 4092
+    },
+    {
+      "epoch": 0.02890969838693279,
+      "grad_norm": 1.3979246616363525,
+      "learning_rate": 4.901653461101213e-05,
+      "loss": 0.6505,
+      "step": 4123
+    },
+    {
+      "epoch": 0.029127064540217996,
+      "grad_norm": 1.3866580724716187,
+      "learning_rate": 4.8993075348363626e-05,
+      "loss": 0.6461,
+      "step": 4154
+    },
+    {
+      "epoch": 0.029344430693503207,
+      "grad_norm": 1.4662801027297974,
+      "learning_rate": 4.896934532712084e-05,
+      "loss": 0.6448,
+      "step": 4185
+    },
+    {
+      "epoch": 0.029561796846788415,
+      "grad_norm": 1.3171806335449219,
+      "learning_rate": 4.8945344815073846e-05,
+      "loss": 0.6525,
+      "step": 4216
+    },
+    {
+      "epoch": 0.029779163000073623,
+      "grad_norm": 1.3289718627929688,
+      "learning_rate": 4.892107408306516e-05,
+      "loss": 0.6445,
+      "step": 4247
+    },
+    {
+      "epoch": 0.029996529153358834,
+      "grad_norm": 1.3527250289916992,
+      "learning_rate": 4.889653340498669e-05,
+      "loss": 0.6449,
+      "step": 4278
+    },
+    {
+      "epoch": 0.03021389530664404,
+      "grad_norm": 1.314674735069275,
+      "learning_rate": 4.8871723057776664e-05,
+      "loss": 0.6559,
+      "step": 4309
+    },
+    {
+      "epoch": 0.030431261459929252,
+      "grad_norm": 1.2237507104873657,
+      "learning_rate": 4.8846643321416476e-05,
+      "loss": 0.6468,
+      "step": 4340
+    },
+    {
+      "epoch": 0.03064862761321446,
+      "grad_norm": 1.423965573310852,
+      "learning_rate": 4.882129447892753e-05,
+      "loss": 0.6465,
+      "step": 4371
+    },
+    {
+      "epoch": 0.030865993766499667,
+      "grad_norm": 1.271371603012085,
+      "learning_rate": 4.8795676816368076e-05,
+      "loss": 0.6471,
+      "step": 4402
+    },
+    {
+      "epoch": 0.031083359919784878,
+      "grad_norm": 1.5492364168167114,
+      "learning_rate": 4.876979062282995e-05,
+      "loss": 0.6444,
+      "step": 4433
+    },
+    {
+      "epoch": 0.031300726073070086,
+      "grad_norm": 1.5139328241348267,
+      "learning_rate": 4.8743636190435325e-05,
+      "loss": 0.6512,
+      "step": 4464
+    },
+    {
+      "epoch": 0.03151809222635529,
+      "grad_norm": 1.4510358572006226,
+      "learning_rate": 4.871721381433344e-05,
+      "loss": 0.6485,
+      "step": 4495
+    },
+    {
+      "epoch": 0.03173545837964051,
+      "grad_norm": 1.3228867053985596,
+      "learning_rate": 4.869052379269719e-05,
+      "loss": 0.6416,
+      "step": 4526
+    },
+    {
+      "epoch": 0.031952824532925715,
+      "grad_norm": 1.2731959819793701,
+      "learning_rate": 4.866356642671985e-05,
+      "loss": 0.6477,
+      "step": 4557
+    },
+    {
+      "epoch": 0.03217019068621092,
+      "grad_norm": 1.2540090084075928,
+      "learning_rate": 4.8636342020611634e-05,
+      "loss": 0.6412,
+      "step": 4588
+    },
+    {
+      "epoch": 0.03238755683949613,
+      "grad_norm": 1.662154197692871,
+      "learning_rate": 4.860885088159626e-05,
+      "loss": 0.6464,
+      "step": 4619
+    },
+    {
+      "epoch": 0.03260492299278134,
+      "grad_norm": 1.5419702529907227,
+      "learning_rate": 4.858109331990751e-05,
+      "loss": 0.6421,
+      "step": 4650
+    },
+    {
+      "epoch": 0.03282228914606655,
+      "grad_norm": 1.6409112215042114,
+      "learning_rate": 4.855306964878567e-05,
+      "loss": 0.6468,
+      "step": 4681
+    },
+    {
+      "epoch": 0.03303965529935176,
+      "grad_norm": 1.1416597366333008,
+      "learning_rate": 4.8524780184474084e-05,
+      "loss": 0.6396,
+      "step": 4712
+    },
+    {
+      "epoch": 0.03325702145263697,
+      "grad_norm": 1.215846061706543,
+      "learning_rate": 4.8496225246215496e-05,
+      "loss": 0.6517,
+      "step": 4743
+    },
+    {
+      "epoch": 0.033474387605922175,
+      "grad_norm": 1.3075084686279297,
+      "learning_rate": 4.8467405156248505e-05,
+      "loss": 0.6389,
+      "step": 4774
+    },
+    {
+      "epoch": 0.03369175375920738,
+      "grad_norm": 1.6089972257614136,
+      "learning_rate": 4.843832023980392e-05,
+      "loss": 0.6371,
+      "step": 4805
+    },
+    {
+      "epoch": 0.0339091199124926,
+      "grad_norm": 1.3927685022354126,
+      "learning_rate": 4.840897082510106e-05,
+      "loss": 0.6355,
+      "step": 4836
+    },
+    {
+      "epoch": 0.034126486065777804,
+      "grad_norm": 1.197952389717102,
+      "learning_rate": 4.8379357243344084e-05,
+      "loss": 0.6415,
+      "step": 4867
+    },
+    {
+      "epoch": 0.03434385221906301,
+      "grad_norm": 1.2738877534866333,
+      "learning_rate": 4.8349479828718236e-05,
+      "loss": 0.6442,
+      "step": 4898
+    },
+    {
+      "epoch": 0.03456121837234822,
+      "grad_norm": 1.5164271593093872,
+      "learning_rate": 4.8319338918386075e-05,
+      "loss": 0.6406,
+      "step": 4929
+    },
+    {
+      "epoch": 0.03477858452563343,
+      "grad_norm": 1.242473840713501,
+      "learning_rate": 4.828893485248369e-05,
+      "loss": 0.6441,
+      "step": 4960
+    },
+    {
+      "epoch": 0.03499595067891864,
+      "grad_norm": 1.2919869422912598,
+      "learning_rate": 4.825826797411682e-05,
+      "loss": 0.6487,
+      "step": 4991
+    },
+    {
+      "epoch": 0.03521331683220385,
+      "grad_norm": 2.6493895053863525,
+      "learning_rate": 4.822733862935702e-05,
+      "loss": 0.6398,
+      "step": 5022
+    },
+    {
+      "epoch": 0.035430682985489056,
+      "grad_norm": 1.2134305238723755,
+      "learning_rate": 4.819614716723775e-05,
+      "loss": 0.6398,
+      "step": 5053
+    },
+    {
+      "epoch": 0.035648049138774264,
+      "grad_norm": 1.1468703746795654,
+      "learning_rate": 4.8164693939750425e-05,
+      "loss": 0.633,
+      "step": 5084
+    },
+    {
+      "epoch": 0.03586541529205947,
+      "grad_norm": 1.3923726081848145,
+      "learning_rate": 4.813297930184042e-05,
+      "loss": 0.6295,
+      "step": 5115
+    },
+    {
+      "epoch": 0.036082781445344686,
+      "grad_norm": 1.5531644821166992,
+      "learning_rate": 4.810100361140314e-05,
+      "loss": 0.6345,
+      "step": 5146
+    },
+    {
+      "epoch": 0.036300147598629894,
+      "grad_norm": 1.201889991760254,
+      "learning_rate": 4.8068767229279885e-05,
+      "loss": 0.6414,
+      "step": 5177
+    },
+    {
+      "epoch": 0.0365175137519151,
+      "grad_norm": 1.4365577697753906,
+      "learning_rate": 4.8036270519253854e-05,
+      "loss": 0.6364,
+      "step": 5208
+    },
+    {
+      "epoch": 0.03673487990520031,
+      "grad_norm": 1.210980772972107,
+      "learning_rate": 4.8003513848046e-05,
+      "loss": 0.6353,
+      "step": 5239
+    },
+    {
+      "epoch": 0.036952246058485516,
+      "grad_norm": 1.379381775856018,
+      "learning_rate": 4.79704975853109e-05,
+      "loss": 0.6293,
+      "step": 5270
+    },
+    {
+      "epoch": 0.03716961221177073,
+      "grad_norm": 1.2263178825378418,
+      "learning_rate": 4.793722210363262e-05,
+      "loss": 0.629,
+      "step": 5301
+    },
+    {
+      "epoch": 0.03738697836505594,
+      "grad_norm": 1.2448405027389526,
+      "learning_rate": 4.7903687778520414e-05,
+      "loss": 0.6474,
+      "step": 5332
+    },
+    {
+      "epoch": 0.037604344518341146,
+      "grad_norm": 1.257132887840271,
+      "learning_rate": 4.7869894988404593e-05,
+      "loss": 0.6426,
+      "step": 5363
+    },
+    {
+      "epoch": 0.03782171067162635,
+      "grad_norm": 1.3763643503189087,
+      "learning_rate": 4.783584411463221e-05,
+      "loss": 0.6396,
+      "step": 5394
+    },
+    {
+      "epoch": 0.03803907682491156,
+      "grad_norm": 1.3164068460464478,
+      "learning_rate": 4.780153554146274e-05,
+      "loss": 0.6343,
+      "step": 5425
+    },
+    {
+      "epoch": 0.038256442978196775,
+      "grad_norm": 1.2034872770309448,
+      "learning_rate": 4.7766969656063766e-05,
+      "loss": 0.629,
+      "step": 5456
+    },
+    {
+      "epoch": 0.03847380913148198,
+      "grad_norm": 1.3156630992889404,
+      "learning_rate": 4.773214684850662e-05,
+      "loss": 0.6403,
+      "step": 5487
+    },
+    {
+      "epoch": 0.03869117528476719,
+      "grad_norm": 1.0719150304794312,
+      "learning_rate": 4.769706751176193e-05,
+      "loss": 0.6318,
+      "step": 5518
+    },
+    {
+      "epoch": 0.0389085414380524,
+      "grad_norm": 1.3054882287979126,
+      "learning_rate": 4.7661732041695264e-05,
+      "loss": 0.6419,
+      "step": 5549
+    },
+    {
+      "epoch": 0.039125907591337605,
+      "grad_norm": 1.2081729173660278,
+      "learning_rate": 4.762614083706258e-05,
+      "loss": 0.6461,
+      "step": 5580
+    },
+    {
+      "epoch": 0.03934327374462282,
+      "grad_norm": 1.1728904247283936,
+      "learning_rate": 4.759029429950581e-05,
+      "loss": 0.6473,
+      "step": 5611
+    },
+    {
+      "epoch": 0.03956063989790803,
+      "grad_norm": 1.2552399635314941,
+      "learning_rate": 4.7554192833548235e-05,
+      "loss": 0.6373,
+      "step": 5642
+    },
+    {
+      "epoch": 0.039778006051193235,
+      "grad_norm": 1.299212098121643,
+      "learning_rate": 4.751783684659e-05,
+      "loss": 0.6234,
+      "step": 5673
+    },
+    {
+      "epoch": 0.03999537220447844,
+      "grad_norm": 2.0746827125549316,
+      "learning_rate": 4.748122674890348e-05,
+      "loss": 0.6264,
+      "step": 5704
+    },
+    {
+      "epoch": 0.04021273835776365,
+      "grad_norm": 1.3712407350540161,
+      "learning_rate": 4.7444362953628654e-05,
+      "loss": 0.6252,
+      "step": 5735
+    },
+    {
+      "epoch": 0.040430104511048864,
+      "grad_norm": 1.2094186544418335,
+      "learning_rate": 4.7407245876768424e-05,
+      "loss": 0.6265,
+      "step": 5766
+    },
+    {
+      "epoch": 0.04064747066433407,
+      "grad_norm": 1.2487757205963135,
+      "learning_rate": 4.736987593718397e-05,
+      "loss": 0.6305,
+      "step": 5797
+    },
+    {
+      "epoch": 0.04086483681761928,
+      "grad_norm": 1.5805151462554932,
+      "learning_rate": 4.733225355658999e-05,
+      "loss": 0.631,
+      "step": 5828
+    },
+    {
+      "epoch": 0.04108220297090449,
+      "grad_norm": 1.2615118026733398,
+      "learning_rate": 4.7294379159549926e-05,
+      "loss": 0.636,
+      "step": 5859
+    },
+    {
+      "epoch": 0.041299569124189695,
+      "grad_norm": 1.6970707178115845,
+      "learning_rate": 4.725625317347119e-05,
+      "loss": 0.6271,
+      "step": 5890
+    },
+    {
+      "epoch": 0.04151693527747491,
+      "grad_norm": 1.4279624223709106,
+      "learning_rate": 4.7217876028600374e-05,
+      "loss": 0.6267,
+      "step": 5921
+    },
+    {
+      "epoch": 0.041734301430760117,
+      "grad_norm": 1.3471580743789673,
+      "learning_rate": 4.717924815801832e-05,
+      "loss": 0.6289,
+      "step": 5952
+    },
+    {
+      "epoch": 0.041951667584045324,
+      "grad_norm": 1.5088621377944946,
+      "learning_rate": 4.714036999763532e-05,
+      "loss": 0.6263,
+      "step": 5983
+    },
+    {
+      "epoch": 0.04216903373733053,
+      "grad_norm": 1.3061436414718628,
+      "learning_rate": 4.7101241986186116e-05,
+      "loss": 0.6266,
+      "step": 6014
+    },
+    {
+      "epoch": 0.04238639989061574,
+      "grad_norm": 1.1800014972686768,
+      "learning_rate": 4.7061864565225e-05,
+      "loss": 0.6259,
+      "step": 6045
+    },
+    {
+      "epoch": 0.042603766043900954,
+      "grad_norm": 1.195177435874939,
+      "learning_rate": 4.702223817912081e-05,
+      "loss": 0.6251,
+      "step": 6076
+    },
+    {
+      "epoch": 0.04282113219718616,
+      "grad_norm": 1.1905118227005005,
+      "learning_rate": 4.698236327505195e-05,
+      "loss": 0.6217,
+      "step": 6107
+    },
+    {
+      "epoch": 0.04303849835047137,
+      "grad_norm": 1.2270928621292114,
+      "learning_rate": 4.694224030300127e-05,
+      "loss": 0.6352,
+      "step": 6138
+    },
+    {
+      "epoch": 0.043255864503756576,
+      "grad_norm": 1.209226369857788,
+      "learning_rate": 4.690186971575107e-05,
+      "loss": 0.6249,
+      "step": 6169
+    },
+    {
+      "epoch": 0.043473230657041784,
+      "grad_norm": 1.1949187517166138,
+      "learning_rate": 4.6861251968877916e-05,
+      "loss": 0.6371,
+      "step": 6200
+    },
+    {
+      "epoch": 0.043690596810327,
+      "grad_norm": 1.2346535921096802,
+      "learning_rate": 4.68203875207476e-05,
+      "loss": 0.6177,
+      "step": 6231
+    },
+    {
+      "epoch": 0.043907962963612206,
+      "grad_norm": 1.2187124490737915,
+      "learning_rate": 4.677927683250983e-05,
+      "loss": 0.6268,
+      "step": 6262
+    },
+    {
+      "epoch": 0.04412532911689741,
+      "grad_norm": 1.2187339067459106,
+      "learning_rate": 4.6737920368093156e-05,
+      "loss": 0.6261,
+      "step": 6293
+    },
+    {
+      "epoch": 0.04434269527018262,
+      "grad_norm": 1.320764422416687,
+      "learning_rate": 4.669631859419965e-05,
+      "loss": 0.6212,
+      "step": 6324
+    },
+    {
+      "epoch": 0.04456006142346783,
+      "grad_norm": 1.3396878242492676,
+      "learning_rate": 4.6654471980299676e-05,
+      "loss": 0.6241,
+      "step": 6355
+    },
+    {
+      "epoch": 0.04477742757675304,
+      "grad_norm": 1.2472412586212158,
+      "learning_rate": 4.661238099862658e-05,
+      "loss": 0.6241,
+      "step": 6386
+    },
+    {
+      "epoch": 0.04499479373003825,
+      "grad_norm": 1.3773880004882812,
+      "learning_rate": 4.657004612417138e-05,
+      "loss": 0.6245,
+      "step": 6417
+    },
+    {
+      "epoch": 0.04521215988332346,
+      "grad_norm": 1.1602933406829834,
+      "learning_rate": 4.6527467834677374e-05,
+      "loss": 0.6221,
+      "step": 6448
+    },
+    {
+      "epoch": 0.045429526036608665,
+      "grad_norm": 1.1719809770584106,
+      "learning_rate": 4.648464661063478e-05,
+      "loss": 0.629,
+      "step": 6479
+    },
+    {
+      "epoch": 0.04564689218989387,
+      "grad_norm": 1.1973191499710083,
+      "learning_rate": 4.6441582935275264e-05,
+      "loss": 0.6239,
+      "step": 6510
+    },
+    {
+      "epoch": 0.04586425834317909,
+      "grad_norm": 1.1805142164230347,
+      "learning_rate": 4.6398277294566586e-05,
+      "loss": 0.6242,
+      "step": 6541
+    },
+    {
+      "epoch": 0.046081624496464295,
+      "grad_norm": 1.3194178342819214,
+      "learning_rate": 4.6354730177207e-05,
+      "loss": 0.6316,
+      "step": 6572
+    },
+    {
+      "epoch": 0.0462989906497495,
+      "grad_norm": 1.1395046710968018,
+      "learning_rate": 4.6310942074619787e-05,
+      "loss": 0.6207,
+      "step": 6603
+    },
+    {
+      "epoch": 0.04651635680303471,
+      "grad_norm": 1.5031641721725464,
+      "learning_rate": 4.626691348094777e-05,
+      "loss": 0.6218,
+      "step": 6634
+    },
+    {
+      "epoch": 0.04673372295631992,
+      "grad_norm": 1.0985206365585327,
+      "learning_rate": 4.622264489304762e-05,
+      "loss": 0.6198,
+      "step": 6665
+    },
+    {
+      "epoch": 0.04695108910960513,
+      "grad_norm": 1.2820847034454346,
+      "learning_rate": 4.617813681048434e-05,
+      "loss": 0.626,
+      "step": 6696
+    },
+    {
+      "epoch": 0.04716845526289034,
+      "grad_norm": 1.4487061500549316,
+      "learning_rate": 4.61333897355256e-05,
+      "loss": 0.622,
+      "step": 6727
+    },
+    {
+      "epoch": 0.04738582141617555,
+      "grad_norm": 1.1577301025390625,
+      "learning_rate": 4.608840417313604e-05,
+      "loss": 0.6244,
+      "step": 6758
+    },
+    {
+      "epoch": 0.047603187569460755,
+      "grad_norm": 1.1363381147384644,
+      "learning_rate": 4.6043180630971646e-05,
+      "loss": 0.6206,
+      "step": 6789
+    },
+    {
+      "epoch": 0.04782055372274597,
+      "grad_norm": 1.22281813621521,
+      "learning_rate": 4.599771961937391e-05,
+      "loss": 0.6231,
+      "step": 6820
+    },
+    {
+      "epoch": 0.048037919876031177,
+      "grad_norm": 1.156031847000122,
+      "learning_rate": 4.5952021651364204e-05,
+      "loss": 0.6223,
+      "step": 6851
+    },
+    {
+      "epoch": 0.048255286029316384,
+      "grad_norm": 1.1688473224639893,
+      "learning_rate": 4.590608724263786e-05,
+      "loss": 0.6293,
+      "step": 6882
+    },
+    {
+      "epoch": 0.04847265218260159,
+      "grad_norm": 1.4265236854553223,
+      "learning_rate": 4.585991691155845e-05,
+      "loss": 0.6109,
+      "step": 6913
+    },
+    {
+      "epoch": 0.0486900183358868,
+      "grad_norm": 1.1242969036102295,
+      "learning_rate": 4.581351117915188e-05,
+      "loss": 0.6256,
+      "step": 6944
+    },
+    {
+      "epoch": 0.048907384489172014,
+      "grad_norm": 1.3943792581558228,
+      "learning_rate": 4.5766870569100534e-05,
+      "loss": 0.6093,
+      "step": 6975
+    },
+    {
+      "epoch": 0.04912475064245722,
+      "grad_norm": 1.0761889219284058,
+      "learning_rate": 4.571999560773736e-05,
+      "loss": 0.6207,
+      "step": 7006
+    },
+    {
+      "epoch": 0.04934211679574243,
+      "grad_norm": 1.2784191370010376,
+      "learning_rate": 4.5672886824039915e-05,
+      "loss": 0.6227,
+      "step": 7037
+    },
+    {
+      "epoch": 0.049559482949027636,
+      "grad_norm": 1.4215577840805054,
+      "learning_rate": 4.5625544749624435e-05,
+      "loss": 0.6148,
+      "step": 7068
+    },
+    {
+      "epoch": 0.049776849102312844,
+      "grad_norm": 1.1120682954788208,
+      "learning_rate": 4.5577969918739794e-05,
+      "loss": 0.6192,
+      "step": 7099
+    },
+    {
+      "epoch": 0.04999421525559806,
+      "grad_norm": 1.1718186140060425,
+      "learning_rate": 4.5530162868261486e-05,
+      "loss": 0.6179,
+      "step": 7130
+    },
+    {
+      "epoch": 0.050211581408883266,
+      "grad_norm": 1.5078628063201904,
+      "learning_rate": 4.548212413768558e-05,
+      "loss": 0.6245,
+      "step": 7161
+    },
+    {
+      "epoch": 0.05042894756216847,
+      "grad_norm": 1.1740144491195679,
+      "learning_rate": 4.543385426912261e-05,
+      "loss": 0.6242,
+      "step": 7192
+    },
+    {
+      "epoch": 0.05064631371545368,
+      "grad_norm": 1.2450133562088013,
+      "learning_rate": 4.53853538072915e-05,
+      "loss": 0.6194,
+      "step": 7223
+    },
+    {
+      "epoch": 0.05086367986873889,
+      "grad_norm": 1.181887149810791,
+      "learning_rate": 4.533662329951336e-05,
+      "loss": 0.6188,
+      "step": 7254
+    },
+    {
+      "epoch": 0.0510810460220241,
+      "grad_norm": 1.172691822052002,
+      "learning_rate": 4.528766329570536e-05,
+      "loss": 0.6191,
+      "step": 7285
+    },
+    {
+      "epoch": 0.05129841217530931,
+      "grad_norm": 1.148863673210144,
+      "learning_rate": 4.523847434837447e-05,
+      "loss": 0.6234,
+      "step": 7316
+    },
+    {
+      "epoch": 0.05151577832859452,
+      "grad_norm": 1.2533507347106934,
+      "learning_rate": 4.518905701261128e-05,
+      "loss": 0.6166,
+      "step": 7347
+    },
+    {
+      "epoch": 0.051733144481879725,
+      "grad_norm": 1.1888121366500854,
+      "learning_rate": 4.5139411846083715e-05,
+      "loss": 0.6176,
+      "step": 7378
+    },
+    {
+      "epoch": 0.05195051063516493,
+      "grad_norm": 1.0766541957855225,
+      "learning_rate": 4.508953940903073e-05,
+      "loss": 0.6137,
+      "step": 7409
+    },
+    {
+      "epoch": 0.05216787678845015,
+      "grad_norm": 1.2277822494506836,
+      "learning_rate": 4.5039440264255994e-05,
+      "loss": 0.6231,
+      "step": 7440
+    },
+    {
+      "epoch": 0.052385242941735355,
+      "grad_norm": 1.0808931589126587,
+      "learning_rate": 4.498911497712155e-05,
+      "loss": 0.6209,
+      "step": 7471
+    },
+    {
+      "epoch": 0.05260260909502056,
+      "grad_norm": 1.0840133428573608,
+      "learning_rate": 4.493856411554142e-05,
+      "loss": 0.6144,
+      "step": 7502
+    },
+    {
+      "epoch": 0.05281997524830577,
+      "grad_norm": 1.5881050825119019,
+      "learning_rate": 4.4887788249975206e-05,
+      "loss": 0.6114,
+      "step": 7533
+    },
+    {
+      "epoch": 0.05303734140159098,
+      "grad_norm": 1.1180490255355835,
+      "learning_rate": 4.4836787953421656e-05,
+      "loss": 0.6134,
+      "step": 7564
+    },
+    {
+      "epoch": 0.05325470755487619,
+      "grad_norm": 1.1469063758850098,
+      "learning_rate": 4.478556380141218e-05,
+      "loss": 0.6122,
+      "step": 7595
+    },
+    {
+      "epoch": 0.0534720737081614,
+      "grad_norm": 1.2977004051208496,
+      "learning_rate": 4.4734116372004375e-05,
+      "loss": 0.615,
+      "step": 7626
+    },
+    {
+      "epoch": 0.05368943986144661,
+      "grad_norm": 1.0556434392929077,
+      "learning_rate": 4.4682446245775477e-05,
+      "loss": 0.6187,
+      "step": 7657
+    },
+    {
+      "epoch": 0.053906806014731815,
+      "grad_norm": 1.101298451423645,
+      "learning_rate": 4.463055400581586e-05,
+      "loss": 0.6123,
+      "step": 7688
+    },
+    {
+      "epoch": 0.05412417216801702,
+      "grad_norm": 1.262608528137207,
+      "learning_rate": 4.4578440237722374e-05,
+      "loss": 0.6149,
+      "step": 7719
+    },
+    {
+      "epoch": 0.05434153832130224,
+      "grad_norm": 1.1554538011550903,
+      "learning_rate": 4.452610552959183e-05,
+      "loss": 0.6168,
+      "step": 7750
+    },
+    {
+      "epoch": 0.054558904474587444,
+      "grad_norm": 1.1653157472610474,
+      "learning_rate": 4.447355047201428e-05,
+      "loss": 0.615,
+      "step": 7781
+    },
+    {
+      "epoch": 0.05477627062787265,
+      "grad_norm": 1.0511231422424316,
+      "learning_rate": 4.4420775658066414e-05,
+      "loss": 0.6215,
+      "step": 7812
+    },
+    {
+      "epoch": 0.05499363678115786,
+      "grad_norm": 1.3468266725540161,
+      "learning_rate": 4.436778168330484e-05,
+      "loss": 0.6221,
+      "step": 7843
+    },
+    {
+      "epoch": 0.05521100293444307,
+      "grad_norm": 1.3238797187805176,
+      "learning_rate": 4.4314569145759353e-05,
+      "loss": 0.611,
+      "step": 7874
+    },
+    {
+      "epoch": 0.05542836908772828,
+      "grad_norm": 1.1107763051986694,
+      "learning_rate": 4.42611386459262e-05,
+      "loss": 0.6119,
+      "step": 7905
+    },
+    {
+      "epoch": 0.05564573524101349,
+      "grad_norm": 1.3667259216308594,
+      "learning_rate": 4.420749078676133e-05,
+      "loss": 0.6134,
+      "step": 7936
+    },
+    {
+      "epoch": 0.055863101394298696,
+      "grad_norm": 1.0336949825286865,
+      "learning_rate": 4.4153626173673516e-05,
+      "loss": 0.613,
+      "step": 7967
+    },
+    {
+      "epoch": 0.056080467547583904,
+      "grad_norm": 1.0911316871643066,
+      "learning_rate": 4.409954541451762e-05,
+      "loss": 0.6107,
+      "step": 7998
+    },
+    {
+      "epoch": 0.05629783370086911,
+      "grad_norm": 1.2516382932662964,
+      "learning_rate": 4.404524911958764e-05,
+      "loss": 0.6138,
+      "step": 8029
+    },
+    {
+      "epoch": 0.056515199854154326,
+      "grad_norm": 1.1678277254104614,
+      "learning_rate": 4.399073790160989e-05,
+      "loss": 0.6162,
+      "step": 8060
+    },
+    {
+      "epoch": 0.05673256600743953,
+      "grad_norm": 1.4072996377944946,
+      "learning_rate": 4.393601237573607e-05,
+      "loss": 0.614,
+      "step": 8091
+    },
+    {
+      "epoch": 0.05694993216072474,
+      "grad_norm": 1.2312837839126587,
+      "learning_rate": 4.388107315953628e-05,
+      "loss": 0.6203,
+      "step": 8122
+    },
+    {
+      "epoch": 0.05716729831400995,
+      "grad_norm": 1.1720649003982544,
+      "learning_rate": 4.382592087299212e-05,
+      "loss": 0.619,
+      "step": 8153
+    },
+    {
+      "epoch": 0.057384664467295156,
+      "grad_norm": 1.0711950063705444,
+      "learning_rate": 4.377055613848964e-05,
+      "loss": 0.6124,
+      "step": 8184
+    },
+    {
+      "epoch": 0.05760203062058037,
+      "grad_norm": 1.1664263010025024,
+      "learning_rate": 4.3714979580812355e-05,
+      "loss": 0.6073,
+      "step": 8215
+    },
+    {
+      "epoch": 0.05781939677386558,
+      "grad_norm": 0.9937831163406372,
+      "learning_rate": 4.365919182713416e-05,
+      "loss": 0.6101,
+      "step": 8246
+    },
+    {
+      "epoch": 0.058036762927150785,
+      "grad_norm": 1.0545841455459595,
+      "learning_rate": 4.360319350701226e-05,
+      "loss": 0.6089,
+      "step": 8277
+    },
+    {
+      "epoch": 0.05825412908043599,
+      "grad_norm": 1.120007038116455,
+      "learning_rate": 4.3546985252380115e-05,
+      "loss": 0.6078,
+      "step": 8308
+    },
+    {
+      "epoch": 0.0584714952337212,
+      "grad_norm": 1.1087831258773804,
+      "learning_rate": 4.349056769754021e-05,
+      "loss": 0.601,
+      "step": 8339
+    },
+    {
+      "epoch": 0.058688861387006415,
+      "grad_norm": 1.1370675563812256,
+      "learning_rate": 4.3433941479156994e-05,
+      "loss": 0.6103,
+      "step": 8370
+    },
+    {
+      "epoch": 0.05890622754029162,
+      "grad_norm": 1.182654857635498,
+      "learning_rate": 4.3377107236249647e-05,
+      "loss": 0.6115,
+      "step": 8401
+    },
+    {
+      "epoch": 0.05912359369357683,
+      "grad_norm": 1.2606432437896729,
+      "learning_rate": 4.332006561018488e-05,
+      "loss": 0.6139,
+      "step": 8432
+    },
+    {
+      "epoch": 0.05934095984686204,
+      "grad_norm": 1.1618047952651978,
+      "learning_rate": 4.3262817244669683e-05,
+      "loss": 0.6053,
+      "step": 8463
+    },
+    {
+      "epoch": 0.059558326000147245,
+      "grad_norm": 1.1281821727752686,
+      "learning_rate": 4.3205362785744083e-05,
+      "loss": 0.6084,
+      "step": 8494
+    },
+    {
+      "epoch": 0.05977569215343246,
+      "grad_norm": 1.15248703956604,
+      "learning_rate": 4.314770288177384e-05,
+      "loss": 0.6128,
+      "step": 8525
+    },
+    {
+      "epoch": 0.05999305830671767,
+      "grad_norm": 1.2170960903167725,
+      "learning_rate": 4.308983818344313e-05,
+      "loss": 0.6134,
+      "step": 8556
+    },
+    {
+      "epoch": 0.060210424460002875,
+      "grad_norm": 1.085260033607483,
+      "learning_rate": 4.3031769343747206e-05,
+      "loss": 0.6212,
+      "step": 8587
+    },
+    {
+      "epoch": 0.06042779061328808,
+      "grad_norm": 1.2243624925613403,
+      "learning_rate": 4.297349701798505e-05,
+      "loss": 0.605,
+      "step": 8618
+    },
+    {
+      "epoch": 0.06064515676657329,
+      "grad_norm": 1.124769687652588,
+      "learning_rate": 4.2915021863751916e-05,
+      "loss": 0.6074,
+      "step": 8649
+    },
+    {
+      "epoch": 0.060862522919858504,
+      "grad_norm": 1.893545150756836,
+      "learning_rate": 4.285634454093198e-05,
+      "loss": 0.6055,
+      "step": 8680
+    },
+    {
+      "epoch": 0.06107988907314371,
+      "grad_norm": 1.2186870574951172,
+      "learning_rate": 4.279746571169086e-05,
+      "loss": 0.6121,
+      "step": 8711
+    },
+    {
+      "epoch": 0.06129725522642892,
+      "grad_norm": 1.0006957054138184,
+      "learning_rate": 4.2738386040468136e-05,
+      "loss": 0.6079,
+      "step": 8742
+    },
+    {
+      "epoch": 0.06151462137971413,
+      "grad_norm": 1.0913995504379272,
+      "learning_rate": 4.2679106193969866e-05,
+      "loss": 0.6107,
+      "step": 8773
+    },
+    {
+      "epoch": 0.061731987532999334,
+      "grad_norm": 1.4647611379623413,
+      "learning_rate": 4.261962684116106e-05,
+      "loss": 0.6188,
+      "step": 8804
+    },
+    {
+      "epoch": 0.06194935368628455,
+      "grad_norm": 1.0805052518844604,
+      "learning_rate": 4.2559948653258145e-05,
+      "loss": 0.6137,
+      "step": 8835
+    },
+    {
+      "epoch": 0.062166719839569756,
+      "grad_norm": 1.0512675046920776,
+      "learning_rate": 4.250007230372134e-05,
+      "loss": 0.6052,
+      "step": 8866
+    },
+    {
+      "epoch": 0.062384085992854964,
+      "grad_norm": 1.1758863925933838,
+      "learning_rate": 4.2439998468247126e-05,
+      "loss": 0.6148,
+      "step": 8897
+    },
+    {
+      "epoch": 0.06260145214614017,
+      "grad_norm": 1.0526351928710938,
+      "learning_rate": 4.2379727824760566e-05,
+      "loss": 0.5973,
+      "step": 8928
+    },
+    {
+      "epoch": 0.06281881829942539,
+      "grad_norm": 1.077563762664795,
+      "learning_rate": 4.231926105340768e-05,
+      "loss": 0.6114,
+      "step": 8959
+    },
+    {
+      "epoch": 0.06303618445271059,
+      "grad_norm": 1.0116938352584839,
+      "learning_rate": 4.225859883654776e-05,
+      "loss": 0.6146,
+      "step": 8990
+    },
+    {
+      "epoch": 0.0632535506059958,
+      "grad_norm": 1.1967271566390991,
+      "learning_rate": 4.219774185874569e-05,
+      "loss": 0.5972,
+      "step": 9021
+    },
+    {
+      "epoch": 0.06347091675928102,
+      "grad_norm": 1.1610004901885986,
+      "learning_rate": 4.213669080676418e-05,
+      "loss": 0.6081,
+      "step": 9052
+    },
+    {
+      "epoch": 0.06368828291256622,
+      "grad_norm": 1.0253360271453857,
+      "learning_rate": 4.2075446369556056e-05,
+      "loss": 0.6118,
+      "step": 9083
+    },
+    {
+      "epoch": 0.06390564906585143,
+      "grad_norm": 1.140599012374878,
+      "learning_rate": 4.201400923825648e-05,
+      "loss": 0.6091,
+      "step": 9114
+    },
+    {
+      "epoch": 0.06412301521913663,
+      "grad_norm": 1.0821545124053955,
+      "learning_rate": 4.195238010617511e-05,
+      "loss": 0.6028,
+      "step": 9145
+    },
+    {
+      "epoch": 0.06434038137242185,
+      "grad_norm": 1.2161924839019775,
+      "learning_rate": 4.1890559668788344e-05,
+      "loss": 0.6113,
+      "step": 9176
+    },
+    {
+      "epoch": 0.06455774752570706,
+      "grad_norm": 1.128986120223999,
+      "learning_rate": 4.1828548623731405e-05,
+      "loss": 0.6125,
+      "step": 9207
+    },
+    {
+      "epoch": 0.06477511367899226,
+      "grad_norm": 1.0566622018814087,
+      "learning_rate": 4.1766347670790506e-05,
+      "loss": 0.6055,
+      "step": 9238
+    },
+    {
+      "epoch": 0.06499247983227747,
+      "grad_norm": 1.0849272012710571,
+      "learning_rate": 4.170395751189495e-05,
+      "loss": 0.6059,
+      "step": 9269
+    },
+    {
+      "epoch": 0.06520984598556268,
+      "grad_norm": 1.2344416379928589,
+      "learning_rate": 4.164137885110921e-05,
+      "loss": 0.6081,
+      "step": 9300
+    },
+    {
+      "epoch": 0.06542721213884789,
+      "grad_norm": 1.0132557153701782,
+      "learning_rate": 4.157861239462495e-05,
+      "loss": 0.6103,
+      "step": 9331
+    },
+    {
+      "epoch": 0.0656445782921331,
+      "grad_norm": 1.0176981687545776,
+      "learning_rate": 4.1515658850753114e-05,
+      "loss": 0.6086,
+      "step": 9362
+    },
+    {
+      "epoch": 0.0658619444454183,
+      "grad_norm": 1.2429553270339966,
+      "learning_rate": 4.145251892991588e-05,
+      "loss": 0.6091,
+      "step": 9393
+    },
+    {
+      "epoch": 0.06607931059870352,
+      "grad_norm": 1.1481480598449707,
+      "learning_rate": 4.138919334463868e-05,
+      "loss": 0.6048,
+      "step": 9424
+    },
+    {
+      "epoch": 0.06629667675198872,
+      "grad_norm": 0.9623891711235046,
+      "learning_rate": 4.1325682809542124e-05,
+      "loss": 0.5995,
+      "step": 9455
+    },
+    {
+      "epoch": 0.06651404290527393,
+      "grad_norm": 1.2031786441802979,
+      "learning_rate": 4.126198804133398e-05,
+      "loss": 0.6008,
+      "step": 9486
+    },
+    {
+      "epoch": 0.06673140905855915,
+      "grad_norm": 1.3027530908584595,
+      "learning_rate": 4.1198109758801055e-05,
+      "loss": 0.5993,
+      "step": 9517
+    },
+    {
+      "epoch": 0.06694877521184435,
+      "grad_norm": 1.1712346076965332,
+      "learning_rate": 4.113404868280107e-05,
+      "loss": 0.5996,
+      "step": 9548
+    },
+    {
+      "epoch": 0.06716614136512956,
+      "grad_norm": 1.1397625207901,
+      "learning_rate": 4.106980553625457e-05,
+      "loss": 0.5979,
+      "step": 9579
+    },
+    {
+      "epoch": 0.06738350751841476,
+      "grad_norm": 1.122983694076538,
+      "learning_rate": 4.100538104413674e-05,
+      "loss": 0.599,
+      "step": 9610
+    },
+    {
+      "epoch": 0.06760087367169998,
+      "grad_norm": 1.0782618522644043,
+      "learning_rate": 4.09407759334692e-05,
+      "loss": 0.6005,
+      "step": 9641
+    },
+    {
+      "epoch": 0.0678182398249852,
+      "grad_norm": 1.0563392639160156,
+      "learning_rate": 4.087599093331186e-05,
+      "loss": 0.6105,
+      "step": 9672
+    },
+    {
+      "epoch": 0.0680356059782704,
+      "grad_norm": 1.0684717893600464,
+      "learning_rate": 4.081102677475462e-05,
+      "loss": 0.6092,
+      "step": 9703
+    },
+    {
+      "epoch": 0.06825297213155561,
+      "grad_norm": 1.152811884880066,
+      "learning_rate": 4.0745884190909194e-05,
+      "loss": 0.6035,
+      "step": 9734
+    },
+    {
+      "epoch": 0.06847033828484081,
+      "grad_norm": 1.1413112878799438,
+      "learning_rate": 4.0680563916900796e-05,
+      "loss": 0.6018,
+      "step": 9765
+    },
+    {
+      "epoch": 0.06868770443812602,
+      "grad_norm": 0.981604814529419,
+      "learning_rate": 4.0615066689859815e-05,
+      "loss": 0.6035,
+      "step": 9796
+    },
+    {
+      "epoch": 0.06890507059141124,
+      "grad_norm": 1.0713250637054443,
+      "learning_rate": 4.0549393248913584e-05,
+      "loss": 0.597,
+      "step": 9827
+    },
+    {
+      "epoch": 0.06912243674469644,
+      "grad_norm": 0.949456512928009,
+      "learning_rate": 4.048354433517794e-05,
+      "loss": 0.6063,
+      "step": 9858
+    },
+    {
+      "epoch": 0.06933980289798165,
+      "grad_norm": 1.0702073574066162,
+      "learning_rate": 4.0417520691748916e-05,
+      "loss": 0.5963,
+      "step": 9889
+    },
+    {
+      "epoch": 0.06955716905126685,
+      "grad_norm": 0.9974942803382874,
+      "learning_rate": 4.035132306369438e-05,
+      "loss": 0.611,
+      "step": 9920
+    },
+    {
+      "epoch": 0.06977453520455207,
+      "grad_norm": 1.0833642482757568,
+      "learning_rate": 4.028495219804555e-05,
+      "loss": 0.6014,
+      "step": 9951
+    },
+    {
+      "epoch": 0.06999190135783728,
+      "grad_norm": 1.181322693824768,
+      "learning_rate": 4.021840884378864e-05,
+      "loss": 0.599,
+      "step": 9982
+    },
+    {
+      "epoch": 0.07020926751112248,
+      "grad_norm": 1.0088189840316772,
+      "learning_rate": 4.015169375185633e-05,
+      "loss": 0.5955,
+      "step": 10013
+    },
+    {
+      "epoch": 0.0704266336644077,
+      "grad_norm": 1.0920222997665405,
+      "learning_rate": 4.0084807675119396e-05,
+      "loss": 0.5985,
+      "step": 10044
+    },
+    {
+      "epoch": 0.0706439998176929,
+      "grad_norm": 1.0450975894927979,
+      "learning_rate": 4.0017751368378106e-05,
+      "loss": 0.6069,
+      "step": 10075
+    },
+    {
+      "epoch": 0.07086136597097811,
+      "grad_norm": 1.1613439321517944,
+      "learning_rate": 3.995052558835377e-05,
+      "loss": 0.5992,
+      "step": 10106
+    },
+    {
+      "epoch": 0.07107873212426333,
+      "grad_norm": 1.2720811367034912,
+      "learning_rate": 3.988313109368017e-05,
+      "loss": 0.6012,
+      "step": 10137
+    },
+    {
+      "epoch": 0.07129609827754853,
+      "grad_norm": 1.0315334796905518,
+      "learning_rate": 3.981556864489504e-05,
+      "loss": 0.6149,
+      "step": 10168
+    },
+    {
+      "epoch": 0.07151346443083374,
+      "grad_norm": 1.3320891857147217,
+      "learning_rate": 3.974783900443142e-05,
+      "loss": 0.6013,
+      "step": 10199
+    },
+    {
+      "epoch": 0.07173083058411894,
+      "grad_norm": 1.7277195453643799,
+      "learning_rate": 3.9679942936609095e-05,
+      "loss": 0.5983,
+      "step": 10230
+    },
+    {
+      "epoch": 0.07194819673740416,
+      "grad_norm": 1.1056753396987915,
+      "learning_rate": 3.961188120762596e-05,
+      "loss": 0.5952,
+      "step": 10261
+    },
+    {
+      "epoch": 0.07216556289068937,
+      "grad_norm": 1.3461558818817139,
+      "learning_rate": 3.954365458554938e-05,
+      "loss": 0.5975,
+      "step": 10292
+    },
+    {
+      "epoch": 0.07238292904397457,
+      "grad_norm": 1.1405422687530518,
+      "learning_rate": 3.947526384030751e-05,
+      "loss": 0.602,
+      "step": 10323
+    },
+    {
+      "epoch": 0.07260029519725979,
+      "grad_norm": 1.2912482023239136,
+      "learning_rate": 3.9406709743680624e-05,
+      "loss": 0.6014,
+      "step": 10354
+    },
+    {
+      "epoch": 0.07281766135054499,
+      "grad_norm": 1.1548583507537842,
+      "learning_rate": 3.9337993069292366e-05,
+      "loss": 0.6053,
+      "step": 10385
+    },
+    {
+      "epoch": 0.0730350275038302,
+      "grad_norm": 1.0558302402496338,
+      "learning_rate": 3.926911459260109e-05,
+      "loss": 0.6062,
+      "step": 10416
+    },
+    {
+      "epoch": 0.07325239365711542,
+      "grad_norm": 1.1658024787902832,
+      "learning_rate": 3.920007509089102e-05,
+      "loss": 0.6016,
+      "step": 10447
+    },
+    {
+      "epoch": 0.07346975981040062,
+      "grad_norm": 1.1093552112579346,
+      "learning_rate": 3.913087534326357e-05,
+      "loss": 0.6025,
+      "step": 10478
+    },
+    {
+      "epoch": 0.07368712596368583,
+      "grad_norm": 1.0908467769622803,
+      "learning_rate": 3.9061516130628475e-05,
+      "loss": 0.5954,
+      "step": 10509
+    },
+    {
+      "epoch": 0.07390449211697103,
+      "grad_norm": 1.0856796503067017,
+      "learning_rate": 3.8991998235695025e-05,
+      "loss": 0.5983,
+      "step": 10540
+    },
+    {
+      "epoch": 0.07412185827025625,
+      "grad_norm": 1.0330291986465454,
+      "learning_rate": 3.8922322442963224e-05,
+      "loss": 0.5955,
+      "step": 10571
+    },
+    {
+      "epoch": 0.07433922442354146,
+      "grad_norm": 0.9688258171081543,
+      "learning_rate": 3.885248953871491e-05,
+      "loss": 0.6001,
+      "step": 10602
+    },
+    {
+      "epoch": 0.07455659057682666,
+      "grad_norm": 1.1808383464813232,
+      "learning_rate": 3.8782500311004915e-05,
+      "loss": 0.5996,
+      "step": 10633
+    },
+    {
+      "epoch": 0.07477395673011188,
+      "grad_norm": 1.1267834901809692,
+      "learning_rate": 3.871235554965218e-05,
+      "loss": 0.5962,
+      "step": 10664
+    },
+    {
+      "epoch": 0.07499132288339708,
+      "grad_norm": 1.1001136302947998,
+      "learning_rate": 3.864205604623078e-05,
+      "loss": 0.6031,
+      "step": 10695
+    },
+    {
+      "epoch": 0.07520868903668229,
+      "grad_norm": 1.009621024131775,
+      "learning_rate": 3.857160259406107e-05,
+      "loss": 0.5976,
+      "step": 10726
+    },
+    {
+      "epoch": 0.0754260551899675,
+      "grad_norm": 0.933811366558075,
+      "learning_rate": 3.8500995988200674e-05,
+      "loss": 0.604,
+      "step": 10757
+    },
+    {
+      "epoch": 0.0756434213432527,
+      "grad_norm": 1.2187680006027222,
+      "learning_rate": 3.843023702543556e-05,
+      "loss": 0.5983,
+      "step": 10788
+    },
+    {
+      "epoch": 0.07586078749653792,
+      "grad_norm": 1.0286976099014282,
+      "learning_rate": 3.8359326504270984e-05,
+      "loss": 0.596,
+      "step": 10819
+    },
+    {
+      "epoch": 0.07607815364982312,
+      "grad_norm": 1.0752719640731812,
+      "learning_rate": 3.828826522492255e-05,
+      "loss": 0.596,
+      "step": 10850
+    },
+    {
+      "epoch": 0.07629551980310834,
+      "grad_norm": 1.1018482446670532,
+      "learning_rate": 3.821705398930713e-05,
+      "loss": 0.603,
+      "step": 10881
+    },
+    {
+      "epoch": 0.07651288595639355,
+      "grad_norm": 0.980965793132782,
+      "learning_rate": 3.814569360103385e-05,
+      "loss": 0.6021,
+      "step": 10912
+    },
+    {
+      "epoch": 0.07673025210967875,
+      "grad_norm": 0.9588106870651245,
+      "learning_rate": 3.807418486539499e-05,
+      "loss": 0.6017,
+      "step": 10943
+    },
+    {
+      "epoch": 0.07694761826296397,
+      "grad_norm": 1.2048044204711914,
+      "learning_rate": 3.80025285893569e-05,
+      "loss": 0.6049,
+      "step": 10974
+    },
+    {
+      "epoch": 0.07716498441624917,
+      "grad_norm": 1.0373165607452393,
+      "learning_rate": 3.793072558155093e-05,
+      "loss": 0.5914,
+      "step": 11005
+    },
+    {
+      "epoch": 0.07738235056953438,
+      "grad_norm": 1.0112730264663696,
+      "learning_rate": 3.785877665226426e-05,
+      "loss": 0.5949,
+      "step": 11036
+    },
+    {
+      "epoch": 0.0775997167228196,
+      "grad_norm": 1.0219500064849854,
+      "learning_rate": 3.778668261343079e-05,
+      "loss": 0.5918,
+      "step": 11067
+    },
+    {
+      "epoch": 0.0778170828761048,
+      "grad_norm": 1.1104828119277954,
+      "learning_rate": 3.771444427862192e-05,
+      "loss": 0.6029,
+      "step": 11098
+    },
+    {
+      "epoch": 0.07803444902939001,
+      "grad_norm": 0.9852742552757263,
+      "learning_rate": 3.7642062463037465e-05,
+      "loss": 0.5839,
+      "step": 11129
+    },
+    {
+      "epoch": 0.07825181518267521,
+      "grad_norm": 1.0593205690383911,
+      "learning_rate": 3.7569537983496373e-05,
+      "loss": 0.5966,
+      "step": 11160
+    },
+    {
+      "epoch": 0.07846918133596043,
+      "grad_norm": 1.2858383655548096,
+      "learning_rate": 3.749687165842753e-05,
+      "loss": 0.5939,
+      "step": 11191
+    },
+    {
+      "epoch": 0.07868654748924564,
+      "grad_norm": 1.2006022930145264,
+      "learning_rate": 3.7424064307860536e-05,
+      "loss": 0.5965,
+      "step": 11222
+    },
+    {
+      "epoch": 0.07890391364253084,
+      "grad_norm": 0.9741569757461548,
+      "learning_rate": 3.735111675341645e-05,
+      "loss": 0.5971,
+      "step": 11253
+    },
+    {
+      "epoch": 0.07912127979581605,
+      "grad_norm": 1.0724074840545654,
+      "learning_rate": 3.7278029818298524e-05,
+      "loss": 0.5901,
+      "step": 11284
+    },
+    {
+      "epoch": 0.07933864594910126,
+      "grad_norm": 1.102207064628601,
+      "learning_rate": 3.720480432728287e-05,
+      "loss": 0.602,
+      "step": 11315
+    },
+    {
+      "epoch": 0.07955601210238647,
+      "grad_norm": 1.0976966619491577,
+      "learning_rate": 3.71314411067092e-05,
+      "loss": 0.6002,
+      "step": 11346
+    },
+    {
+      "epoch": 0.07977337825567168,
+      "grad_norm": 1.0534948110580444,
+      "learning_rate": 3.70579409844715e-05,
+      "loss": 0.5801,
+      "step": 11377
+    },
+    {
+      "epoch": 0.07999074440895688,
+      "grad_norm": 1.1666125059127808,
+      "learning_rate": 3.698430479000865e-05,
+      "loss": 0.5992,
+      "step": 11408
+    },
+    {
+      "epoch": 0.0802081105622421,
+      "grad_norm": 1.0316112041473389,
+      "learning_rate": 3.691053335429509e-05,
+      "loss": 0.5942,
+      "step": 11439
+    },
+    {
+      "epoch": 0.0804254767155273,
+      "grad_norm": 1.0085538625717163,
+      "learning_rate": 3.683662750983147e-05,
+      "loss": 0.5978,
+      "step": 11470
+    },
+    {
+      "epoch": 0.08064284286881251,
+      "grad_norm": 1.0875978469848633,
+      "learning_rate": 3.676258809063518e-05,
+      "loss": 0.5949,
+      "step": 11501
+    },
+    {
+      "epoch": 0.08086020902209773,
+      "grad_norm": 1.1905012130737305,
+      "learning_rate": 3.6688415932231004e-05,
+      "loss": 0.6018,
+      "step": 11532
+    },
+    {
+      "epoch": 0.08107757517538293,
+      "grad_norm": 1.1717733144760132,
+      "learning_rate": 3.661411187164166e-05,
+      "loss": 0.5986,
+      "step": 11563
+    },
+    {
+      "epoch": 0.08129494132866814,
+      "grad_norm": 1.053768515586853,
+      "learning_rate": 3.65396767473784e-05,
+      "loss": 0.5831,
+      "step": 11594
+    },
+    {
+      "epoch": 0.08151230748195334,
+      "grad_norm": 1.095425009727478,
+      "learning_rate": 3.6465111399431465e-05,
+      "loss": 0.5967,
+      "step": 11625
+    },
+    {
+      "epoch": 0.08172967363523856,
+      "grad_norm": 1.005953311920166,
+      "learning_rate": 3.6390416669260674e-05,
+      "loss": 0.5834,
+      "step": 11656
+    },
+    {
+      "epoch": 0.08194703978852377,
+      "grad_norm": 1.0991450548171997,
+      "learning_rate": 3.63155933997859e-05,
+      "loss": 0.5951,
+      "step": 11687
+    },
+    {
+      "epoch": 0.08216440594180897,
+      "grad_norm": 1.1069071292877197,
+      "learning_rate": 3.624064243537758e-05,
+      "loss": 0.5989,
+      "step": 11718
+    },
+    {
+      "epoch": 0.08238177209509419,
+      "grad_norm": 1.099542498588562,
+      "learning_rate": 3.616556462184716e-05,
+      "loss": 0.5949,
+      "step": 11749
+    },
+    {
+      "epoch": 0.08259913824837939,
+      "grad_norm": 1.0950508117675781,
+      "learning_rate": 3.609036080643755e-05,
+      "loss": 0.5911,
+      "step": 11780
+    },
+    {
+      "epoch": 0.0828165044016646,
+      "grad_norm": 0.9489970803260803,
+      "learning_rate": 3.60150318378136e-05,
+      "loss": 0.5922,
+      "step": 11811
+    },
+    {
+      "epoch": 0.08303387055494982,
+      "grad_norm": 0.9877674579620361,
+      "learning_rate": 3.5939578566052465e-05,
+      "loss": 0.5927,
+      "step": 11842
+    },
+    {
+      "epoch": 0.08325123670823502,
+      "grad_norm": 1.060177206993103,
+      "learning_rate": 3.586400184263408e-05,
+      "loss": 0.5899,
+      "step": 11873
+    },
+    {
+      "epoch": 0.08346860286152023,
+      "grad_norm": 1.0531491041183472,
+      "learning_rate": 3.578830252043148e-05,
+      "loss": 0.6004,
+      "step": 11904
+    },
+    {
+      "epoch": 0.08368596901480543,
+      "grad_norm": 0.9791742563247681,
+      "learning_rate": 3.571248145370125e-05,
+      "loss": 0.5926,
+      "step": 11935
+    },
+    {
+      "epoch": 0.08390333516809065,
+      "grad_norm": 1.0492030382156372,
+      "learning_rate": 3.5636539498073794e-05,
+      "loss": 0.5914,
+      "step": 11966
+    },
+    {
+      "epoch": 0.08412070132137586,
+      "grad_norm": 1.2035881280899048,
+      "learning_rate": 3.556047751054378e-05,
+      "loss": 0.589,
+      "step": 11997
+    },
+    {
+      "epoch": 0.08433806747466106,
+      "grad_norm": 1.0384137630462646,
+      "learning_rate": 3.548429634946039e-05,
+      "loss": 0.5894,
+      "step": 12028
+    },
+    {
+      "epoch": 0.08455543362794628,
+      "grad_norm": 1.1555323600769043,
+      "learning_rate": 3.540799687451768e-05,
+      "loss": 0.5899,
+      "step": 12059
+    },
+    {
+      "epoch": 0.08477279978123148,
+      "grad_norm": 1.2114530801773071,
+      "learning_rate": 3.533157994674485e-05,
+      "loss": 0.5888,
+      "step": 12090
+    },
+    {
+      "epoch": 0.08499016593451669,
+      "grad_norm": 1.145213007926941,
+      "learning_rate": 3.5255046428496546e-05,
+      "loss": 0.5887,
+      "step": 12121
+    },
+    {
+      "epoch": 0.08520753208780191,
+      "grad_norm": 1.0313454866409302,
+      "learning_rate": 3.517839718344311e-05,
+      "loss": 0.598,
+      "step": 12152
+    },
+    {
+      "epoch": 0.08542489824108711,
+      "grad_norm": 1.018702745437622,
+      "learning_rate": 3.510163307656086e-05,
+      "loss": 0.5931,
+      "step": 12183
+    }
+  ],
+  "logging_steps": 31,
+  "max_steps": 30517,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 3052,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 9.006624315386364e+18,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-12208/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff6a84b6c2a69c365aedcd04f176d62d0993a049ac50b28b1fc7fee54f4267a8
+size 5304

checkpoint-15260/config.json

@@ -0,0 +1,39 @@
+{
+  "_name_or_path": "meta-llama/Llama-3.1-8B-Instruct",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.0.dev0",
+  "use_cache": true,
+  "vocab_size": 128256
+}

checkpoint-15260/generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.44.0.dev0"
+}

checkpoint-15260/model-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4cb026f0a5829683c10c1335673e1436a45abc9b1455d3a942d36da85b766d6
+size 4886466168

checkpoint-15260/model-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23fa5aaf8ac56c17c821980c9aee44fcbc2a617fc570e875e6fdb93d92886c72
+size 4832007448

checkpoint-15260/model-00003-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a223e87240e50dfb05634ea909cf1ec715d37a12b681b14a3a6e97f28688f2b
+size 4999813112

checkpoint-15260/model-00004-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3047d199c611c6d12b1cca780f033ad89df9c076f742674f7ac4a892ca9573b3
+size 4999813128

checkpoint-15260/model-00005-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a2f91d749d664ecd22c71f3d54488966fc6899a03924b8fea3613c8f83398a8
+size 4832007496

checkpoint-15260/model-00006-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17020e4642a219332982712a64e7d359e75890d10bd4f46cc75489e5dda3c81a
+size 4999813120

checkpoint-15260/model-00007-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56927410b72fca2e6df4a7cf12fc42d5aeffb51d50e287009620cbb282635c2d
+size 2571158184

checkpoint-15260/model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 32121044992
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00007-of-00007.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.norm.weight": "model-00007-of-00007.safetensors"
+  }
+}

checkpoint-15260/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3bfc58974f01b39e88901516dd69714dccbc66c4658216919664749de0901d2b
+size 15385036334

checkpoint-15260/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
+size 14244

checkpoint-15260/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ce335347efaebf038b8915bc0a8d2d587a4a1aa08ad0e015b4bc7cc4fba634e
+size 1064

checkpoint-15260/trainer_state.json

@@ -0,0 +1,3477 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.10700024190749317,
+  "eval_steps": 500,
+  "global_step": 15260,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.00021736615328520894,
+      "grad_norm": 5.709163665771484,
+      "learning_rate": 1.0157273918741808e-06,
+      "loss": 0.9228,
+      "step": 31
+    },
+    {
+      "epoch": 0.0004347323065704179,
+      "grad_norm": 3.9758756160736084,
+      "learning_rate": 2.0314547837483616e-06,
+      "loss": 0.8276,
+      "step": 62
+    },
+    {
+      "epoch": 0.0006520984598556268,
+      "grad_norm": 3.721677303314209,
+      "learning_rate": 3.0471821756225426e-06,
+      "loss": 0.7897,
+      "step": 93
+    },
+    {
+      "epoch": 0.0008694646131408358,
+      "grad_norm": 3.384953498840332,
+      "learning_rate": 4.062909567496723e-06,
+      "loss": 0.7641,
+      "step": 124
+    },
+    {
+      "epoch": 0.0010868307664260446,
+      "grad_norm": 3.19750714302063,
+      "learning_rate": 5.078636959370905e-06,
+      "loss": 0.758,
+      "step": 155
+    },
+    {
+      "epoch": 0.0013041969197112536,
+      "grad_norm": 3.49003267288208,
+      "learning_rate": 6.094364351245085e-06,
+      "loss": 0.7644,
+      "step": 186
+    },
+    {
+      "epoch": 0.0015215630729964625,
+      "grad_norm": 3.323367118835449,
+      "learning_rate": 7.110091743119267e-06,
+      "loss": 0.7606,
+      "step": 217
+    },
+    {
+      "epoch": 0.0017389292262816715,
+      "grad_norm": 3.465822219848633,
+      "learning_rate": 8.125819134993446e-06,
+      "loss": 0.7505,
+      "step": 248
+    },
+    {
+      "epoch": 0.0019562953795668804,
+      "grad_norm": 2.7967450618743896,
+      "learning_rate": 9.141546526867629e-06,
+      "loss": 0.7593,
+      "step": 279
+    },
+    {
+      "epoch": 0.002173661532852089,
+      "grad_norm": 3.5493738651275635,
+      "learning_rate": 1.015727391874181e-05,
+      "loss": 0.7514,
+      "step": 310
+    },
+    {
+      "epoch": 0.0023910276861372984,
+      "grad_norm": 3.514606237411499,
+      "learning_rate": 1.117300131061599e-05,
+      "loss": 0.7497,
+      "step": 341
+    },
+    {
+      "epoch": 0.0026083938394225073,
+      "grad_norm": 3.0246028900146484,
+      "learning_rate": 1.218872870249017e-05,
+      "loss": 0.7473,
+      "step": 372
+    },
+    {
+      "epoch": 0.002825759992707716,
+      "grad_norm": 3.01147723197937,
+      "learning_rate": 1.3204456094364351e-05,
+      "loss": 0.739,
+      "step": 403
+    },
+    {
+      "epoch": 0.003043126145992925,
+      "grad_norm": 3.566333293914795,
+      "learning_rate": 1.4220183486238533e-05,
+      "loss": 0.7447,
+      "step": 434
+    },
+    {
+      "epoch": 0.003260492299278134,
+      "grad_norm": 3.8698947429656982,
+      "learning_rate": 1.5235910878112714e-05,
+      "loss": 0.7447,
+      "step": 465
+    },
+    {
+      "epoch": 0.003477858452563343,
+      "grad_norm": 2.567028760910034,
+      "learning_rate": 1.6251638269986893e-05,
+      "loss": 0.7496,
+      "step": 496
+    },
+    {
+      "epoch": 0.003695224605848552,
+      "grad_norm": 2.8029377460479736,
+      "learning_rate": 1.7267365661861077e-05,
+      "loss": 0.7324,
+      "step": 527
+    },
+    {
+      "epoch": 0.003912590759133761,
+      "grad_norm": 2.862530469894409,
+      "learning_rate": 1.8283093053735257e-05,
+      "loss": 0.7371,
+      "step": 558
+    },
+    {
+      "epoch": 0.00412995691241897,
+      "grad_norm": 2.9063901901245117,
+      "learning_rate": 1.9298820445609438e-05,
+      "loss": 0.7398,
+      "step": 589
+    },
+    {
+      "epoch": 0.004347323065704178,
+      "grad_norm": 3.187814235687256,
+      "learning_rate": 2.031454783748362e-05,
+      "loss": 0.7362,
+      "step": 620
+    },
+    {
+      "epoch": 0.004564689218989388,
+      "grad_norm": 2.6158969402313232,
+      "learning_rate": 2.13302752293578e-05,
+      "loss": 0.7405,
+      "step": 651
+    },
+    {
+      "epoch": 0.004782055372274597,
+      "grad_norm": 2.702423334121704,
+      "learning_rate": 2.234600262123198e-05,
+      "loss": 0.7313,
+      "step": 682
+    },
+    {
+      "epoch": 0.004999421525559805,
+      "grad_norm": 2.88313889503479,
+      "learning_rate": 2.336173001310616e-05,
+      "loss": 0.7337,
+      "step": 713
+    },
+    {
+      "epoch": 0.0052167876788450146,
+      "grad_norm": 2.8978841304779053,
+      "learning_rate": 2.437745740498034e-05,
+      "loss": 0.73,
+      "step": 744
+    },
+    {
+      "epoch": 0.005434153832130223,
+      "grad_norm": 2.791414976119995,
+      "learning_rate": 2.5393184796854525e-05,
+      "loss": 0.7297,
+      "step": 775
+    },
+    {
+      "epoch": 0.005651519985415432,
+      "grad_norm": 2.432821035385132,
+      "learning_rate": 2.6408912188728702e-05,
+      "loss": 0.7326,
+      "step": 806
+    },
+    {
+      "epoch": 0.0058688861387006415,
+      "grad_norm": 2.430279493331909,
+      "learning_rate": 2.7424639580602886e-05,
+      "loss": 0.7333,
+      "step": 837
+    },
+    {
+      "epoch": 0.00608625229198585,
+      "grad_norm": 2.5513761043548584,
+      "learning_rate": 2.8440366972477066e-05,
+      "loss": 0.7296,
+      "step": 868
+    },
+    {
+      "epoch": 0.006303618445271059,
+      "grad_norm": 2.478562116622925,
+      "learning_rate": 2.9456094364351244e-05,
+      "loss": 0.7303,
+      "step": 899
+    },
+    {
+      "epoch": 0.006520984598556268,
+      "grad_norm": 2.5496723651885986,
+      "learning_rate": 3.0471821756225428e-05,
+      "loss": 0.7303,
+      "step": 930
+    },
+    {
+      "epoch": 0.006738350751841477,
+      "grad_norm": 2.114060640335083,
+      "learning_rate": 3.148754914809961e-05,
+      "loss": 0.7184,
+      "step": 961
+    },
+    {
+      "epoch": 0.006955716905126686,
+      "grad_norm": 2.325977325439453,
+      "learning_rate": 3.2503276539973785e-05,
+      "loss": 0.7268,
+      "step": 992
+    },
+    {
+      "epoch": 0.0071730830584118945,
+      "grad_norm": 2.0549662113189697,
+      "learning_rate": 3.351900393184797e-05,
+      "loss": 0.724,
+      "step": 1023
+    },
+    {
+      "epoch": 0.007390449211697104,
+      "grad_norm": 1.913522481918335,
+      "learning_rate": 3.453473132372215e-05,
+      "loss": 0.7206,
+      "step": 1054
+    },
+    {
+      "epoch": 0.007607815364982313,
+      "grad_norm": 2.0651443004608154,
+      "learning_rate": 3.555045871559633e-05,
+      "loss": 0.7239,
+      "step": 1085
+    },
+    {
+      "epoch": 0.007825181518267521,
+      "grad_norm": 2.2482309341430664,
+      "learning_rate": 3.6566186107470514e-05,
+      "loss": 0.728,
+      "step": 1116
+    },
+    {
+      "epoch": 0.00804254767155273,
+      "grad_norm": 2.349695920944214,
+      "learning_rate": 3.7581913499344695e-05,
+      "loss": 0.721,
+      "step": 1147
+    },
+    {
+      "epoch": 0.00825991382483794,
+      "grad_norm": 2.218843460083008,
+      "learning_rate": 3.8597640891218876e-05,
+      "loss": 0.7248,
+      "step": 1178
+    },
+    {
+      "epoch": 0.00847727997812315,
+      "grad_norm": 2.0839340686798096,
+      "learning_rate": 3.9613368283093056e-05,
+      "loss": 0.7233,
+      "step": 1209
+    },
+    {
+      "epoch": 0.008694646131408357,
+      "grad_norm": 2.0757343769073486,
+      "learning_rate": 4.062909567496724e-05,
+      "loss": 0.7166,
+      "step": 1240
+    },
+    {
+      "epoch": 0.008912012284693566,
+      "grad_norm": 2.052342653274536,
+      "learning_rate": 4.164482306684142e-05,
+      "loss": 0.7256,
+      "step": 1271
+    },
+    {
+      "epoch": 0.009129378437978775,
+      "grad_norm": 2.1202704906463623,
+      "learning_rate": 4.26605504587156e-05,
+      "loss": 0.7131,
+      "step": 1302
+    },
+    {
+      "epoch": 0.009346744591263985,
+      "grad_norm": 2.1945958137512207,
+      "learning_rate": 4.367627785058978e-05,
+      "loss": 0.708,
+      "step": 1333
+    },
+    {
+      "epoch": 0.009564110744549194,
+      "grad_norm": 2.106307029724121,
+      "learning_rate": 4.469200524246396e-05,
+      "loss": 0.7139,
+      "step": 1364
+    },
+    {
+      "epoch": 0.009781476897834401,
+      "grad_norm": 2.3779594898223877,
+      "learning_rate": 4.570773263433814e-05,
+      "loss": 0.7205,
+      "step": 1395
+    },
+    {
+      "epoch": 0.00999884305111961,
+      "grad_norm": 2.001551866531372,
+      "learning_rate": 4.672346002621232e-05,
+      "loss": 0.7162,
+      "step": 1426
+    },
+    {
+      "epoch": 0.01021620920440482,
+      "grad_norm": 2.6994752883911133,
+      "learning_rate": 4.77391874180865e-05,
+      "loss": 0.7215,
+      "step": 1457
+    },
+    {
+      "epoch": 0.010433575357690029,
+      "grad_norm": 2.071122884750366,
+      "learning_rate": 4.875491480996068e-05,
+      "loss": 0.7095,
+      "step": 1488
+    },
+    {
+      "epoch": 0.010650941510975238,
+      "grad_norm": 2.3666610717773438,
+      "learning_rate": 4.977064220183487e-05,
+      "loss": 0.7089,
+      "step": 1519
+    },
+    {
+      "epoch": 0.010868307664260446,
+      "grad_norm": 2.130204677581787,
+      "learning_rate": 4.9999915451558777e-05,
+      "loss": 0.7043,
+      "step": 1550
+    },
+    {
+      "epoch": 0.011085673817545655,
+      "grad_norm": 1.7413716316223145,
+      "learning_rate": 4.999955597496219e-05,
+      "loss": 0.7062,
+      "step": 1581
+    },
+    {
+      "epoch": 0.011303039970830864,
+      "grad_norm": 1.7087843418121338,
+      "learning_rate": 4.9998914381774255e-05,
+      "loss": 0.7175,
+      "step": 1612
+    },
+    {
+      "epoch": 0.011520406124116074,
+      "grad_norm": 2.574871301651001,
+      "learning_rate": 4.999799067923527e-05,
+      "loss": 0.7076,
+      "step": 1643
+    },
+    {
+      "epoch": 0.011737772277401283,
+      "grad_norm": 2.0530433654785156,
+      "learning_rate": 4.999678487776908e-05,
+      "loss": 0.708,
+      "step": 1674
+    },
+    {
+      "epoch": 0.011955138430686492,
+      "grad_norm": 1.9184463024139404,
+      "learning_rate": 4.9995296990983006e-05,
+      "loss": 0.7011,
+      "step": 1705
+    },
+    {
+      "epoch": 0.0121725045839717,
+      "grad_norm": 1.768115520477295,
+      "learning_rate": 4.999352703566763e-05,
+      "loss": 0.6995,
+      "step": 1736
+    },
+    {
+      "epoch": 0.012389870737256909,
+      "grad_norm": 1.6692063808441162,
+      "learning_rate": 4.999147503179668e-05,
+      "loss": 0.7019,
+      "step": 1767
+    },
+    {
+      "epoch": 0.012607236890542118,
+      "grad_norm": 1.803944706916809,
+      "learning_rate": 4.998914100252672e-05,
+      "loss": 0.703,
+      "step": 1798
+    },
+    {
+      "epoch": 0.012824603043827328,
+      "grad_norm": 1.8957322835922241,
+      "learning_rate": 4.998652497419696e-05,
+      "loss": 0.6979,
+      "step": 1829
+    },
+    {
+      "epoch": 0.013041969197112537,
+      "grad_norm": 1.544054627418518,
+      "learning_rate": 4.9983626976328927e-05,
+      "loss": 0.7012,
+      "step": 1860
+    },
+    {
+      "epoch": 0.013259335350397744,
+      "grad_norm": 1.7066351175308228,
+      "learning_rate": 4.998044704162613e-05,
+      "loss": 0.6949,
+      "step": 1891
+    },
+    {
+      "epoch": 0.013476701503682954,
+      "grad_norm": 1.7045214176177979,
+      "learning_rate": 4.9976985205973705e-05,
+      "loss": 0.6871,
+      "step": 1922
+    },
+    {
+      "epoch": 0.013694067656968163,
+      "grad_norm": 2.0414485931396484,
+      "learning_rate": 4.997324150843799e-05,
+      "loss": 0.6944,
+      "step": 1953
+    },
+    {
+      "epoch": 0.013911433810253372,
+      "grad_norm": 1.9210485219955444,
+      "learning_rate": 4.99692159912661e-05,
+      "loss": 0.7013,
+      "step": 1984
+    },
+    {
+      "epoch": 0.014128799963538581,
+      "grad_norm": 1.8733997344970703,
+      "learning_rate": 4.996490869988546e-05,
+      "loss": 0.6991,
+      "step": 2015
+    },
+    {
+      "epoch": 0.014346166116823789,
+      "grad_norm": 1.7118934392929077,
+      "learning_rate": 4.996031968290326e-05,
+      "loss": 0.6873,
+      "step": 2046
+    },
+    {
+      "epoch": 0.014563532270108998,
+      "grad_norm": 1.889208197593689,
+      "learning_rate": 4.995544899210594e-05,
+      "loss": 0.692,
+      "step": 2077
+    },
+    {
+      "epoch": 0.014780898423394207,
+      "grad_norm": 2.0081522464752197,
+      "learning_rate": 4.9950296682458583e-05,
+      "loss": 0.6823,
+      "step": 2108
+    },
+    {
+      "epoch": 0.014998264576679417,
+      "grad_norm": 1.7950328588485718,
+      "learning_rate": 4.994486281210429e-05,
+      "loss": 0.686,
+      "step": 2139
+    },
+    {
+      "epoch": 0.015215630729964626,
+      "grad_norm": 1.758333444595337,
+      "learning_rate": 4.9939147442363566e-05,
+      "loss": 0.6827,
+      "step": 2170
+    },
+    {
+      "epoch": 0.015432996883249834,
+      "grad_norm": 1.496063470840454,
+      "learning_rate": 4.9933150637733574e-05,
+      "loss": 0.6896,
+      "step": 2201
+    },
+    {
+      "epoch": 0.015650363036535043,
+      "grad_norm": 1.6103190183639526,
+      "learning_rate": 4.992687246588743e-05,
+      "loss": 0.6865,
+      "step": 2232
+    },
+    {
+      "epoch": 0.015867729189820254,
+      "grad_norm": 1.7069604396820068,
+      "learning_rate": 4.992031299767347e-05,
+      "loss": 0.6836,
+      "step": 2263
+    },
+    {
+      "epoch": 0.01608509534310546,
+      "grad_norm": 1.8261148929595947,
+      "learning_rate": 4.9913472307114386e-05,
+      "loss": 0.692,
+      "step": 2294
+    },
+    {
+      "epoch": 0.01630246149639067,
+      "grad_norm": 1.7522642612457275,
+      "learning_rate": 4.9906350471406446e-05,
+      "loss": 0.6832,
+      "step": 2325
+    },
+    {
+      "epoch": 0.01651982764967588,
+      "grad_norm": 1.6746042966842651,
+      "learning_rate": 4.989894757091861e-05,
+      "loss": 0.6829,
+      "step": 2356
+    },
+    {
+      "epoch": 0.016737193802961087,
+      "grad_norm": 1.6785473823547363,
+      "learning_rate": 4.989126368919158e-05,
+      "loss": 0.6842,
+      "step": 2387
+    },
+    {
+      "epoch": 0.0169545599562463,
+      "grad_norm": 1.936916470527649,
+      "learning_rate": 4.988329891293693e-05,
+      "loss": 0.6798,
+      "step": 2418
+    },
+    {
+      "epoch": 0.017171926109531506,
+      "grad_norm": 1.5104129314422607,
+      "learning_rate": 4.987505333203608e-05,
+      "loss": 0.6743,
+      "step": 2449
+    },
+    {
+      "epoch": 0.017389292262816713,
+      "grad_norm": 1.3770678043365479,
+      "learning_rate": 4.9866527039539276e-05,
+      "loss": 0.6874,
+      "step": 2480
+    },
+    {
+      "epoch": 0.017606658416101924,
+      "grad_norm": 1.4900861978530884,
+      "learning_rate": 4.9857720131664594e-05,
+      "loss": 0.679,
+      "step": 2511
+    },
+    {
+      "epoch": 0.017824024569387132,
+      "grad_norm": 1.5600172281265259,
+      "learning_rate": 4.9848632707796773e-05,
+      "loss": 0.6719,
+      "step": 2542
+    },
+    {
+      "epoch": 0.018041390722672343,
+      "grad_norm": 1.660237431526184,
+      "learning_rate": 4.9839264870486155e-05,
+      "loss": 0.6773,
+      "step": 2573
+    },
+    {
+      "epoch": 0.01825875687595755,
+      "grad_norm": 1.4618791341781616,
+      "learning_rate": 4.9829616725447526e-05,
+      "loss": 0.6767,
+      "step": 2604
+    },
+    {
+      "epoch": 0.018476123029242758,
+      "grad_norm": 1.5567563772201538,
+      "learning_rate": 4.981968838155888e-05,
+      "loss": 0.6706,
+      "step": 2635
+    },
+    {
+      "epoch": 0.01869348918252797,
+      "grad_norm": 1.4879790544509888,
+      "learning_rate": 4.980947995086024e-05,
+      "loss": 0.676,
+      "step": 2666
+    },
+    {
+      "epoch": 0.018910855335813177,
+      "grad_norm": 1.5527766942977905,
+      "learning_rate": 4.979899154855234e-05,
+      "loss": 0.6734,
+      "step": 2697
+    },
+    {
+      "epoch": 0.019128221489098388,
+      "grad_norm": 1.7922642230987549,
+      "learning_rate": 4.9788223292995386e-05,
+      "loss": 0.6813,
+      "step": 2728
+    },
+    {
+      "epoch": 0.019345587642383595,
+      "grad_norm": 1.6453475952148438,
+      "learning_rate": 4.977717530570768e-05,
+      "loss": 0.6749,
+      "step": 2759
+    },
+    {
+      "epoch": 0.019562953795668803,
+      "grad_norm": 1.4713648557662964,
+      "learning_rate": 4.976584771136425e-05,
+      "loss": 0.6761,
+      "step": 2790
+    },
+    {
+      "epoch": 0.019780319948954014,
+      "grad_norm": 1.5513399839401245,
+      "learning_rate": 4.975424063779547e-05,
+      "loss": 0.6739,
+      "step": 2821
+    },
+    {
+      "epoch": 0.01999768610223922,
+      "grad_norm": 1.431796908378601,
+      "learning_rate": 4.974235421598557e-05,
+      "loss": 0.6639,
+      "step": 2852
+    },
+    {
+      "epoch": 0.020215052255524432,
+      "grad_norm": 1.4959752559661865,
+      "learning_rate": 4.973018858007122e-05,
+      "loss": 0.662,
+      "step": 2883
+    },
+    {
+      "epoch": 0.02043241840880964,
+      "grad_norm": 1.4675205945968628,
+      "learning_rate": 4.9717743867339963e-05,
+      "loss": 0.6659,
+      "step": 2914
+    },
+    {
+      "epoch": 0.020649784562094847,
+      "grad_norm": 1.7111692428588867,
+      "learning_rate": 4.9705020218228695e-05,
+      "loss": 0.6732,
+      "step": 2945
+    },
+    {
+      "epoch": 0.020867150715380058,
+      "grad_norm": 1.542748212814331,
+      "learning_rate": 4.969201777632205e-05,
+      "loss": 0.6666,
+      "step": 2976
+    },
+    {
+      "epoch": 0.021084516868665266,
+      "grad_norm": 1.5013272762298584,
+      "learning_rate": 4.9678736688350846e-05,
+      "loss": 0.667,
+      "step": 3007
+    },
+    {
+      "epoch": 0.021301883021950477,
+      "grad_norm": 1.57589852809906,
+      "learning_rate": 4.966517710419033e-05,
+      "loss": 0.6668,
+      "step": 3038
+    },
+    {
+      "epoch": 0.021519249175235684,
+      "grad_norm": 1.612727403640747,
+      "learning_rate": 4.965133917685858e-05,
+      "loss": 0.6653,
+      "step": 3069
+    },
+    {
+      "epoch": 0.021736615328520892,
+      "grad_norm": 1.5110771656036377,
+      "learning_rate": 4.9637223062514714e-05,
+      "loss": 0.6694,
+      "step": 3100
+    },
+    {
+      "epoch": 0.021953981481806103,
+      "grad_norm": 1.5836044549942017,
+      "learning_rate": 4.962282892045718e-05,
+      "loss": 0.6634,
+      "step": 3131
+    },
+    {
+      "epoch": 0.02217134763509131,
+      "grad_norm": 1.5767654180526733,
+      "learning_rate": 4.9608156913121904e-05,
+      "loss": 0.6633,
+      "step": 3162
+    },
+    {
+      "epoch": 0.02238871378837652,
+      "grad_norm": 1.6058984994888306,
+      "learning_rate": 4.959320720608049e-05,
+      "loss": 0.6646,
+      "step": 3193
+    },
+    {
+      "epoch": 0.02260607994166173,
+      "grad_norm": 1.4564005136489868,
+      "learning_rate": 4.9577979968038354e-05,
+      "loss": 0.663,
+      "step": 3224
+    },
+    {
+      "epoch": 0.022823446094946936,
+      "grad_norm": 1.5274450778961182,
+      "learning_rate": 4.956247537083282e-05,
+      "loss": 0.6554,
+      "step": 3255
+    },
+    {
+      "epoch": 0.023040812248232147,
+      "grad_norm": 1.524122953414917,
+      "learning_rate": 4.9546693589431145e-05,
+      "loss": 0.6666,
+      "step": 3286
+    },
+    {
+      "epoch": 0.023258178401517355,
+      "grad_norm": 1.5017430782318115,
+      "learning_rate": 4.9530634801928595e-05,
+      "loss": 0.6661,
+      "step": 3317
+    },
+    {
+      "epoch": 0.023475544554802566,
+      "grad_norm": 1.4247208833694458,
+      "learning_rate": 4.9514299189546395e-05,
+      "loss": 0.6563,
+      "step": 3348
+    },
+    {
+      "epoch": 0.023692910708087773,
+      "grad_norm": 1.3451093435287476,
+      "learning_rate": 4.949768693662973e-05,
+      "loss": 0.6555,
+      "step": 3379
+    },
+    {
+      "epoch": 0.023910276861372984,
+      "grad_norm": 1.418442726135254,
+      "learning_rate": 4.948079823064559e-05,
+      "loss": 0.6587,
+      "step": 3410
+    },
+    {
+      "epoch": 0.024127643014658192,
+      "grad_norm": 1.5460575819015503,
+      "learning_rate": 4.946363326218074e-05,
+      "loss": 0.6602,
+      "step": 3441
+    },
+    {
+      "epoch": 0.0243450091679434,
+      "grad_norm": 1.4053966999053955,
+      "learning_rate": 4.9446192224939525e-05,
+      "loss": 0.6578,
+      "step": 3472
+    },
+    {
+      "epoch": 0.02456237532122861,
+      "grad_norm": 1.420806646347046,
+      "learning_rate": 4.942847531574167e-05,
+      "loss": 0.6544,
+      "step": 3503
+    },
+    {
+      "epoch": 0.024779741474513818,
+      "grad_norm": 1.3220508098602295,
+      "learning_rate": 4.941048273452008e-05,
+      "loss": 0.6642,
+      "step": 3534
+    },
+    {
+      "epoch": 0.02499710762779903,
+      "grad_norm": 1.3819468021392822,
+      "learning_rate": 4.9392214684318605e-05,
+      "loss": 0.6534,
+      "step": 3565
+    },
+    {
+      "epoch": 0.025214473781084237,
+      "grad_norm": 1.4046236276626587,
+      "learning_rate": 4.93736713712897e-05,
+      "loss": 0.6547,
+      "step": 3596
+    },
+    {
+      "epoch": 0.025431839934369444,
+      "grad_norm": 1.965153455734253,
+      "learning_rate": 4.9354853004692124e-05,
+      "loss": 0.6582,
+      "step": 3627
+    },
+    {
+      "epoch": 0.025649206087654655,
+      "grad_norm": 1.6758291721343994,
+      "learning_rate": 4.93357597968886e-05,
+      "loss": 0.6536,
+      "step": 3658
+    },
+    {
+      "epoch": 0.025866572240939863,
+      "grad_norm": 1.4467246532440186,
+      "learning_rate": 4.931639196334338e-05,
+      "loss": 0.6564,
+      "step": 3689
+    },
+    {
+      "epoch": 0.026083938394225074,
+      "grad_norm": 1.5305490493774414,
+      "learning_rate": 4.9296749722619826e-05,
+      "loss": 0.6626,
+      "step": 3720
+    },
+    {
+      "epoch": 0.02630130454751028,
+      "grad_norm": 1.3826133012771606,
+      "learning_rate": 4.9276833296377966e-05,
+      "loss": 0.6624,
+      "step": 3751
+    },
+    {
+      "epoch": 0.02651867070079549,
+      "grad_norm": 1.4920278787612915,
+      "learning_rate": 4.925664290937196e-05,
+      "loss": 0.6417,
+      "step": 3782
+    },
+    {
+      "epoch": 0.0267360368540807,
+      "grad_norm": 14.533783912658691,
+      "learning_rate": 4.9236178789447576e-05,
+      "loss": 0.647,
+      "step": 3813
+    },
+    {
+      "epoch": 0.026953403007365907,
+      "grad_norm": 1.4659409523010254,
+      "learning_rate": 4.921544116753962e-05,
+      "loss": 0.6514,
+      "step": 3844
+    },
+    {
+      "epoch": 0.02717076916065112,
+      "grad_norm": 1.4820138216018677,
+      "learning_rate": 4.919443027766935e-05,
+      "loss": 0.6477,
+      "step": 3875
+    },
+    {
+      "epoch": 0.027388135313936326,
+      "grad_norm": 1.2543880939483643,
+      "learning_rate": 4.91731463569418e-05,
+      "loss": 0.6505,
+      "step": 3906
+    },
+    {
+      "epoch": 0.027605501467221533,
+      "grad_norm": 1.640926718711853,
+      "learning_rate": 4.915158964554312e-05,
+      "loss": 0.6488,
+      "step": 3937
+    },
+    {
+      "epoch": 0.027822867620506744,
+      "grad_norm": 1.3990615606307983,
+      "learning_rate": 4.912976038673786e-05,
+      "loss": 0.658,
+      "step": 3968
+    },
+    {
+      "epoch": 0.028040233773791952,
+      "grad_norm": 1.2064067125320435,
+      "learning_rate": 4.9107658826866254e-05,
+      "loss": 0.6487,
+      "step": 3999
+    },
+    {
+      "epoch": 0.028257599927077163,
+      "grad_norm": 1.5103769302368164,
+      "learning_rate": 4.908528521534139e-05,
+      "loss": 0.6569,
+      "step": 4030
+    },
+    {
+      "epoch": 0.02847496608036237,
+      "grad_norm": 1.5314087867736816,
+      "learning_rate": 4.906263980464644e-05,
+      "loss": 0.6638,
+      "step": 4061
+    },
+    {
+      "epoch": 0.028692332233647578,
+      "grad_norm": 1.4619168043136597,
+      "learning_rate": 4.903972285033178e-05,
+      "loss": 0.6505,
+      "step": 4092
+    },
+    {
+      "epoch": 0.02890969838693279,
+      "grad_norm": 1.3979246616363525,
+      "learning_rate": 4.901653461101213e-05,
+      "loss": 0.6505,
+      "step": 4123
+    },
+    {
+      "epoch": 0.029127064540217996,
+      "grad_norm": 1.3866580724716187,
+      "learning_rate": 4.8993075348363626e-05,
+      "loss": 0.6461,
+      "step": 4154
+    },
+    {
+      "epoch": 0.029344430693503207,
+      "grad_norm": 1.4662801027297974,
+      "learning_rate": 4.896934532712084e-05,
+      "loss": 0.6448,
+      "step": 4185
+    },
+    {
+      "epoch": 0.029561796846788415,
+      "grad_norm": 1.3171806335449219,
+      "learning_rate": 4.8945344815073846e-05,
+      "loss": 0.6525,
+      "step": 4216
+    },
+    {
+      "epoch": 0.029779163000073623,
+      "grad_norm": 1.3289718627929688,
+      "learning_rate": 4.892107408306516e-05,
+      "loss": 0.6445,
+      "step": 4247
+    },
+    {
+      "epoch": 0.029996529153358834,
+      "grad_norm": 1.3527250289916992,
+      "learning_rate": 4.889653340498669e-05,
+      "loss": 0.6449,
+      "step": 4278
+    },
+    {
+      "epoch": 0.03021389530664404,
+      "grad_norm": 1.314674735069275,
+      "learning_rate": 4.8871723057776664e-05,
+      "loss": 0.6559,
+      "step": 4309
+    },
+    {
+      "epoch": 0.030431261459929252,
+      "grad_norm": 1.2237507104873657,
+      "learning_rate": 4.8846643321416476e-05,
+      "loss": 0.6468,
+      "step": 4340
+    },
+    {
+      "epoch": 0.03064862761321446,
+      "grad_norm": 1.423965573310852,
+      "learning_rate": 4.882129447892753e-05,
+      "loss": 0.6465,
+      "step": 4371
+    },
+    {
+      "epoch": 0.030865993766499667,
+      "grad_norm": 1.271371603012085,
+      "learning_rate": 4.8795676816368076e-05,
+      "loss": 0.6471,
+      "step": 4402
+    },
+    {
+      "epoch": 0.031083359919784878,
+      "grad_norm": 1.5492364168167114,
+      "learning_rate": 4.876979062282995e-05,
+      "loss": 0.6444,
+      "step": 4433
+    },
+    {
+      "epoch": 0.031300726073070086,
+      "grad_norm": 1.5139328241348267,
+      "learning_rate": 4.8743636190435325e-05,
+      "loss": 0.6512,
+      "step": 4464
+    },
+    {
+      "epoch": 0.03151809222635529,
+      "grad_norm": 1.4510358572006226,
+      "learning_rate": 4.871721381433344e-05,
+      "loss": 0.6485,
+      "step": 4495
+    },
+    {
+      "epoch": 0.03173545837964051,
+      "grad_norm": 1.3228867053985596,
+      "learning_rate": 4.869052379269719e-05,
+      "loss": 0.6416,
+      "step": 4526
+    },
+    {
+      "epoch": 0.031952824532925715,
+      "grad_norm": 1.2731959819793701,
+      "learning_rate": 4.866356642671985e-05,
+      "loss": 0.6477,
+      "step": 4557
+    },
+    {
+      "epoch": 0.03217019068621092,
+      "grad_norm": 1.2540090084075928,
+      "learning_rate": 4.8636342020611634e-05,
+      "loss": 0.6412,
+      "step": 4588
+    },
+    {
+      "epoch": 0.03238755683949613,
+      "grad_norm": 1.662154197692871,
+      "learning_rate": 4.860885088159626e-05,
+      "loss": 0.6464,
+      "step": 4619
+    },
+    {
+      "epoch": 0.03260492299278134,
+      "grad_norm": 1.5419702529907227,
+      "learning_rate": 4.858109331990751e-05,
+      "loss": 0.6421,
+      "step": 4650
+    },
+    {
+      "epoch": 0.03282228914606655,
+      "grad_norm": 1.6409112215042114,
+      "learning_rate": 4.855306964878567e-05,
+      "loss": 0.6468,
+      "step": 4681
+    },
+    {
+      "epoch": 0.03303965529935176,
+      "grad_norm": 1.1416597366333008,
+      "learning_rate": 4.8524780184474084e-05,
+      "loss": 0.6396,
+      "step": 4712
+    },
+    {
+      "epoch": 0.03325702145263697,
+      "grad_norm": 1.215846061706543,
+      "learning_rate": 4.8496225246215496e-05,
+      "loss": 0.6517,
+      "step": 4743
+    },
+    {
+      "epoch": 0.033474387605922175,
+      "grad_norm": 1.3075084686279297,
+      "learning_rate": 4.8467405156248505e-05,
+      "loss": 0.6389,
+      "step": 4774
+    },
+    {
+      "epoch": 0.03369175375920738,
+      "grad_norm": 1.6089972257614136,
+      "learning_rate": 4.843832023980392e-05,
+      "loss": 0.6371,
+      "step": 4805
+    },
+    {
+      "epoch": 0.0339091199124926,
+      "grad_norm": 1.3927685022354126,
+      "learning_rate": 4.840897082510106e-05,
+      "loss": 0.6355,
+      "step": 4836
+    },
+    {
+      "epoch": 0.034126486065777804,
+      "grad_norm": 1.197952389717102,
+      "learning_rate": 4.8379357243344084e-05,
+      "loss": 0.6415,
+      "step": 4867
+    },
+    {
+      "epoch": 0.03434385221906301,
+      "grad_norm": 1.2738877534866333,
+      "learning_rate": 4.8349479828718236e-05,
+      "loss": 0.6442,
+      "step": 4898
+    },
+    {
+      "epoch": 0.03456121837234822,
+      "grad_norm": 1.5164271593093872,
+      "learning_rate": 4.8319338918386075e-05,
+      "loss": 0.6406,
+      "step": 4929
+    },
+    {
+      "epoch": 0.03477858452563343,
+      "grad_norm": 1.242473840713501,
+      "learning_rate": 4.828893485248369e-05,
+      "loss": 0.6441,
+      "step": 4960
+    },
+    {
+      "epoch": 0.03499595067891864,
+      "grad_norm": 1.2919869422912598,
+      "learning_rate": 4.825826797411682e-05,
+      "loss": 0.6487,
+      "step": 4991
+    },
+    {
+      "epoch": 0.03521331683220385,
+      "grad_norm": 2.6493895053863525,
+      "learning_rate": 4.822733862935702e-05,
+      "loss": 0.6398,
+      "step": 5022
+    },
+    {
+      "epoch": 0.035430682985489056,
+      "grad_norm": 1.2134305238723755,
+      "learning_rate": 4.819614716723775e-05,
+      "loss": 0.6398,
+      "step": 5053
+    },
+    {
+      "epoch": 0.035648049138774264,
+      "grad_norm": 1.1468703746795654,
+      "learning_rate": 4.8164693939750425e-05,
+      "loss": 0.633,
+      "step": 5084
+    },
+    {
+      "epoch": 0.03586541529205947,
+      "grad_norm": 1.3923726081848145,
+      "learning_rate": 4.813297930184042e-05,
+      "loss": 0.6295,
+      "step": 5115
+    },
+    {
+      "epoch": 0.036082781445344686,
+      "grad_norm": 1.5531644821166992,
+      "learning_rate": 4.810100361140314e-05,
+      "loss": 0.6345,
+      "step": 5146
+    },
+    {
+      "epoch": 0.036300147598629894,
+      "grad_norm": 1.201889991760254,
+      "learning_rate": 4.8068767229279885e-05,
+      "loss": 0.6414,
+      "step": 5177
+    },
+    {
+      "epoch": 0.0365175137519151,
+      "grad_norm": 1.4365577697753906,
+      "learning_rate": 4.8036270519253854e-05,
+      "loss": 0.6364,
+      "step": 5208
+    },
+    {
+      "epoch": 0.03673487990520031,
+      "grad_norm": 1.210980772972107,
+      "learning_rate": 4.8003513848046e-05,
+      "loss": 0.6353,
+      "step": 5239
+    },
+    {
+      "epoch": 0.036952246058485516,
+      "grad_norm": 1.379381775856018,
+      "learning_rate": 4.79704975853109e-05,
+      "loss": 0.6293,
+      "step": 5270
+    },
+    {
+      "epoch": 0.03716961221177073,
+      "grad_norm": 1.2263178825378418,
+      "learning_rate": 4.793722210363262e-05,
+      "loss": 0.629,
+      "step": 5301
+    },
+    {
+      "epoch": 0.03738697836505594,
+      "grad_norm": 1.2448405027389526,
+      "learning_rate": 4.7903687778520414e-05,
+      "loss": 0.6474,
+      "step": 5332
+    },
+    {
+      "epoch": 0.037604344518341146,
+      "grad_norm": 1.257132887840271,
+      "learning_rate": 4.7869894988404593e-05,
+      "loss": 0.6426,
+      "step": 5363
+    },
+    {
+      "epoch": 0.03782171067162635,
+      "grad_norm": 1.3763643503189087,
+      "learning_rate": 4.783584411463221e-05,
+      "loss": 0.6396,
+      "step": 5394
+    },
+    {
+      "epoch": 0.03803907682491156,
+      "grad_norm": 1.3164068460464478,
+      "learning_rate": 4.780153554146274e-05,
+      "loss": 0.6343,
+      "step": 5425
+    },
+    {
+      "epoch": 0.038256442978196775,
+      "grad_norm": 1.2034872770309448,
+      "learning_rate": 4.7766969656063766e-05,
+      "loss": 0.629,
+      "step": 5456
+    },
+    {
+      "epoch": 0.03847380913148198,
+      "grad_norm": 1.3156630992889404,
+      "learning_rate": 4.773214684850662e-05,
+      "loss": 0.6403,
+      "step": 5487
+    },
+    {
+      "epoch": 0.03869117528476719,
+      "grad_norm": 1.0719150304794312,
+      "learning_rate": 4.769706751176193e-05,
+      "loss": 0.6318,
+      "step": 5518
+    },
+    {
+      "epoch": 0.0389085414380524,
+      "grad_norm": 1.3054882287979126,
+      "learning_rate": 4.7661732041695264e-05,
+      "loss": 0.6419,
+      "step": 5549
+    },
+    {
+      "epoch": 0.039125907591337605,
+      "grad_norm": 1.2081729173660278,
+      "learning_rate": 4.762614083706258e-05,
+      "loss": 0.6461,
+      "step": 5580
+    },
+    {
+      "epoch": 0.03934327374462282,
+      "grad_norm": 1.1728904247283936,
+      "learning_rate": 4.759029429950581e-05,
+      "loss": 0.6473,
+      "step": 5611
+    },
+    {
+      "epoch": 0.03956063989790803,
+      "grad_norm": 1.2552399635314941,
+      "learning_rate": 4.7554192833548235e-05,
+      "loss": 0.6373,
+      "step": 5642
+    },
+    {
+      "epoch": 0.039778006051193235,
+      "grad_norm": 1.299212098121643,
+      "learning_rate": 4.751783684659e-05,
+      "loss": 0.6234,
+      "step": 5673
+    },
+    {
+      "epoch": 0.03999537220447844,
+      "grad_norm": 2.0746827125549316,
+      "learning_rate": 4.748122674890348e-05,
+      "loss": 0.6264,
+      "step": 5704
+    },
+    {
+      "epoch": 0.04021273835776365,
+      "grad_norm": 1.3712407350540161,
+      "learning_rate": 4.7444362953628654e-05,
+      "loss": 0.6252,
+      "step": 5735
+    },
+    {
+      "epoch": 0.040430104511048864,
+      "grad_norm": 1.2094186544418335,
+      "learning_rate": 4.7407245876768424e-05,
+      "loss": 0.6265,
+      "step": 5766
+    },
+    {
+      "epoch": 0.04064747066433407,
+      "grad_norm": 1.2487757205963135,
+      "learning_rate": 4.736987593718397e-05,
+      "loss": 0.6305,
+      "step": 5797
+    },
+    {
+      "epoch": 0.04086483681761928,
+      "grad_norm": 1.5805151462554932,
+      "learning_rate": 4.733225355658999e-05,
+      "loss": 0.631,
+      "step": 5828
+    },
+    {
+      "epoch": 0.04108220297090449,
+      "grad_norm": 1.2615118026733398,
+      "learning_rate": 4.7294379159549926e-05,
+      "loss": 0.636,
+      "step": 5859
+    },
+    {
+      "epoch": 0.041299569124189695,
+      "grad_norm": 1.6970707178115845,
+      "learning_rate": 4.725625317347119e-05,
+      "loss": 0.6271,
+      "step": 5890
+    },
+    {
+      "epoch": 0.04151693527747491,
+      "grad_norm": 1.4279624223709106,
+      "learning_rate": 4.7217876028600374e-05,
+      "loss": 0.6267,
+      "step": 5921
+    },
+    {
+      "epoch": 0.041734301430760117,
+      "grad_norm": 1.3471580743789673,
+      "learning_rate": 4.717924815801832e-05,
+      "loss": 0.6289,
+      "step": 5952
+    },
+    {
+      "epoch": 0.041951667584045324,
+      "grad_norm": 1.5088621377944946,
+      "learning_rate": 4.714036999763532e-05,
+      "loss": 0.6263,
+      "step": 5983
+    },
+    {
+      "epoch": 0.04216903373733053,
+      "grad_norm": 1.3061436414718628,
+      "learning_rate": 4.7101241986186116e-05,
+      "loss": 0.6266,
+      "step": 6014
+    },
+    {
+      "epoch": 0.04238639989061574,
+      "grad_norm": 1.1800014972686768,
+      "learning_rate": 4.7061864565225e-05,
+      "loss": 0.6259,
+      "step": 6045
+    },
+    {
+      "epoch": 0.042603766043900954,
+      "grad_norm": 1.195177435874939,
+      "learning_rate": 4.702223817912081e-05,
+      "loss": 0.6251,
+      "step": 6076
+    },
+    {
+      "epoch": 0.04282113219718616,
+      "grad_norm": 1.1905118227005005,
+      "learning_rate": 4.698236327505195e-05,
+      "loss": 0.6217,
+      "step": 6107
+    },
+    {
+      "epoch": 0.04303849835047137,
+      "grad_norm": 1.2270928621292114,
+      "learning_rate": 4.694224030300127e-05,
+      "loss": 0.6352,
+      "step": 6138
+    },
+    {
+      "epoch": 0.043255864503756576,
+      "grad_norm": 1.209226369857788,
+      "learning_rate": 4.690186971575107e-05,
+      "loss": 0.6249,
+      "step": 6169
+    },
+    {
+      "epoch": 0.043473230657041784,
+      "grad_norm": 1.1949187517166138,
+      "learning_rate": 4.6861251968877916e-05,
+      "loss": 0.6371,
+      "step": 6200
+    },
+    {
+      "epoch": 0.043690596810327,
+      "grad_norm": 1.2346535921096802,
+      "learning_rate": 4.68203875207476e-05,
+      "loss": 0.6177,
+      "step": 6231
+    },
+    {
+      "epoch": 0.043907962963612206,
+      "grad_norm": 1.2187124490737915,
+      "learning_rate": 4.677927683250983e-05,
+      "loss": 0.6268,
+      "step": 6262
+    },
+    {
+      "epoch": 0.04412532911689741,
+      "grad_norm": 1.2187339067459106,
+      "learning_rate": 4.6737920368093156e-05,
+      "loss": 0.6261,
+      "step": 6293
+    },
+    {
+      "epoch": 0.04434269527018262,
+      "grad_norm": 1.320764422416687,
+      "learning_rate": 4.669631859419965e-05,
+      "loss": 0.6212,
+      "step": 6324
+    },
+    {
+      "epoch": 0.04456006142346783,
+      "grad_norm": 1.3396878242492676,
+      "learning_rate": 4.6654471980299676e-05,
+      "loss": 0.6241,
+      "step": 6355
+    },
+    {
+      "epoch": 0.04477742757675304,
+      "grad_norm": 1.2472412586212158,
+      "learning_rate": 4.661238099862658e-05,
+      "loss": 0.6241,
+      "step": 6386
+    },
+    {
+      "epoch": 0.04499479373003825,
+      "grad_norm": 1.3773880004882812,
+      "learning_rate": 4.657004612417138e-05,
+      "loss": 0.6245,
+      "step": 6417
+    },
+    {
+      "epoch": 0.04521215988332346,
+      "grad_norm": 1.1602933406829834,
+      "learning_rate": 4.6527467834677374e-05,
+      "loss": 0.6221,
+      "step": 6448
+    },
+    {
+      "epoch": 0.045429526036608665,
+      "grad_norm": 1.1719809770584106,
+      "learning_rate": 4.648464661063478e-05,
+      "loss": 0.629,
+      "step": 6479
+    },
+    {
+      "epoch": 0.04564689218989387,
+      "grad_norm": 1.1973191499710083,
+      "learning_rate": 4.6441582935275264e-05,
+      "loss": 0.6239,
+      "step": 6510
+    },
+    {
+      "epoch": 0.04586425834317909,
+      "grad_norm": 1.1805142164230347,
+      "learning_rate": 4.6398277294566586e-05,
+      "loss": 0.6242,
+      "step": 6541
+    },
+    {
+      "epoch": 0.046081624496464295,
+      "grad_norm": 1.3194178342819214,
+      "learning_rate": 4.6354730177207e-05,
+      "loss": 0.6316,
+      "step": 6572
+    },
+    {
+      "epoch": 0.0462989906497495,
+      "grad_norm": 1.1395046710968018,
+      "learning_rate": 4.6310942074619787e-05,
+      "loss": 0.6207,
+      "step": 6603
+    },
+    {
+      "epoch": 0.04651635680303471,
+      "grad_norm": 1.5031641721725464,
+      "learning_rate": 4.626691348094777e-05,
+      "loss": 0.6218,
+      "step": 6634
+    },
+    {
+      "epoch": 0.04673372295631992,
+      "grad_norm": 1.0985206365585327,
+      "learning_rate": 4.622264489304762e-05,
+      "loss": 0.6198,
+      "step": 6665
+    },
+    {
+      "epoch": 0.04695108910960513,
+      "grad_norm": 1.2820847034454346,
+      "learning_rate": 4.617813681048434e-05,
+      "loss": 0.626,
+      "step": 6696
+    },
+    {
+      "epoch": 0.04716845526289034,
+      "grad_norm": 1.4487061500549316,
+      "learning_rate": 4.61333897355256e-05,
+      "loss": 0.622,
+      "step": 6727
+    },
+    {
+      "epoch": 0.04738582141617555,
+      "grad_norm": 1.1577301025390625,
+      "learning_rate": 4.608840417313604e-05,
+      "loss": 0.6244,
+      "step": 6758
+    },
+    {
+      "epoch": 0.047603187569460755,
+      "grad_norm": 1.1363381147384644,
+      "learning_rate": 4.6043180630971646e-05,
+      "loss": 0.6206,
+      "step": 6789
+    },
+    {
+      "epoch": 0.04782055372274597,
+      "grad_norm": 1.22281813621521,
+      "learning_rate": 4.599771961937391e-05,
+      "loss": 0.6231,
+      "step": 6820
+    },
+    {
+      "epoch": 0.048037919876031177,
+      "grad_norm": 1.156031847000122,
+      "learning_rate": 4.5952021651364204e-05,
+      "loss": 0.6223,
+      "step": 6851
+    },
+    {
+      "epoch": 0.048255286029316384,
+      "grad_norm": 1.1688473224639893,
+      "learning_rate": 4.590608724263786e-05,
+      "loss": 0.6293,
+      "step": 6882
+    },
+    {
+      "epoch": 0.04847265218260159,
+      "grad_norm": 1.4265236854553223,
+      "learning_rate": 4.585991691155845e-05,
+      "loss": 0.6109,
+      "step": 6913
+    },
+    {
+      "epoch": 0.0486900183358868,
+      "grad_norm": 1.1242969036102295,
+      "learning_rate": 4.581351117915188e-05,
+      "loss": 0.6256,
+      "step": 6944
+    },
+    {
+      "epoch": 0.048907384489172014,
+      "grad_norm": 1.3943792581558228,
+      "learning_rate": 4.5766870569100534e-05,
+      "loss": 0.6093,
+      "step": 6975
+    },
+    {
+      "epoch": 0.04912475064245722,
+      "grad_norm": 1.0761889219284058,
+      "learning_rate": 4.571999560773736e-05,
+      "loss": 0.6207,
+      "step": 7006
+    },
+    {
+      "epoch": 0.04934211679574243,
+      "grad_norm": 1.2784191370010376,
+      "learning_rate": 4.5672886824039915e-05,
+      "loss": 0.6227,
+      "step": 7037
+    },
+    {
+      "epoch": 0.049559482949027636,
+      "grad_norm": 1.4215577840805054,
+      "learning_rate": 4.5625544749624435e-05,
+      "loss": 0.6148,
+      "step": 7068
+    },
+    {
+      "epoch": 0.049776849102312844,
+      "grad_norm": 1.1120682954788208,
+      "learning_rate": 4.5577969918739794e-05,
+      "loss": 0.6192,
+      "step": 7099
+    },
+    {
+      "epoch": 0.04999421525559806,
+      "grad_norm": 1.1718186140060425,
+      "learning_rate": 4.5530162868261486e-05,
+      "loss": 0.6179,
+      "step": 7130
+    },
+    {
+      "epoch": 0.050211581408883266,
+      "grad_norm": 1.5078628063201904,
+      "learning_rate": 4.548212413768558e-05,
+      "loss": 0.6245,
+      "step": 7161
+    },
+    {
+      "epoch": 0.05042894756216847,
+      "grad_norm": 1.1740144491195679,
+      "learning_rate": 4.543385426912261e-05,
+      "loss": 0.6242,
+      "step": 7192
+    },
+    {
+      "epoch": 0.05064631371545368,
+      "grad_norm": 1.2450133562088013,
+      "learning_rate": 4.53853538072915e-05,
+      "loss": 0.6194,
+      "step": 7223
+    },
+    {
+      "epoch": 0.05086367986873889,
+      "grad_norm": 1.181887149810791,
+      "learning_rate": 4.533662329951336e-05,
+      "loss": 0.6188,
+      "step": 7254
+    },
+    {
+      "epoch": 0.0510810460220241,
+      "grad_norm": 1.172691822052002,
+      "learning_rate": 4.528766329570536e-05,
+      "loss": 0.6191,
+      "step": 7285
+    },
+    {
+      "epoch": 0.05129841217530931,
+      "grad_norm": 1.148863673210144,
+      "learning_rate": 4.523847434837447e-05,
+      "loss": 0.6234,
+      "step": 7316
+    },
+    {
+      "epoch": 0.05151577832859452,
+      "grad_norm": 1.2533507347106934,
+      "learning_rate": 4.518905701261128e-05,
+      "loss": 0.6166,
+      "step": 7347
+    },
+    {
+      "epoch": 0.051733144481879725,
+      "grad_norm": 1.1888121366500854,
+      "learning_rate": 4.5139411846083715e-05,
+      "loss": 0.6176,
+      "step": 7378
+    },
+    {
+      "epoch": 0.05195051063516493,
+      "grad_norm": 1.0766541957855225,
+      "learning_rate": 4.508953940903073e-05,
+      "loss": 0.6137,
+      "step": 7409
+    },
+    {
+      "epoch": 0.05216787678845015,
+      "grad_norm": 1.2277822494506836,
+      "learning_rate": 4.5039440264255994e-05,
+      "loss": 0.6231,
+      "step": 7440
+    },
+    {
+      "epoch": 0.052385242941735355,
+      "grad_norm": 1.0808931589126587,
+      "learning_rate": 4.498911497712155e-05,
+      "loss": 0.6209,
+      "step": 7471
+    },
+    {
+      "epoch": 0.05260260909502056,
+      "grad_norm": 1.0840133428573608,
+      "learning_rate": 4.493856411554142e-05,
+      "loss": 0.6144,
+      "step": 7502
+    },
+    {
+      "epoch": 0.05281997524830577,
+      "grad_norm": 1.5881050825119019,
+      "learning_rate": 4.4887788249975206e-05,
+      "loss": 0.6114,
+      "step": 7533
+    },
+    {
+      "epoch": 0.05303734140159098,
+      "grad_norm": 1.1180490255355835,
+      "learning_rate": 4.4836787953421656e-05,
+      "loss": 0.6134,
+      "step": 7564
+    },
+    {
+      "epoch": 0.05325470755487619,
+      "grad_norm": 1.1469063758850098,
+      "learning_rate": 4.478556380141218e-05,
+      "loss": 0.6122,
+      "step": 7595
+    },
+    {
+      "epoch": 0.0534720737081614,
+      "grad_norm": 1.2977004051208496,
+      "learning_rate": 4.4734116372004375e-05,
+      "loss": 0.615,
+      "step": 7626
+    },
+    {
+      "epoch": 0.05368943986144661,
+      "grad_norm": 1.0556434392929077,
+      "learning_rate": 4.4682446245775477e-05,
+      "loss": 0.6187,
+      "step": 7657
+    },
+    {
+      "epoch": 0.053906806014731815,
+      "grad_norm": 1.101298451423645,
+      "learning_rate": 4.463055400581586e-05,
+      "loss": 0.6123,
+      "step": 7688
+    },
+    {
+      "epoch": 0.05412417216801702,
+      "grad_norm": 1.262608528137207,
+      "learning_rate": 4.4578440237722374e-05,
+      "loss": 0.6149,
+      "step": 7719
+    },
+    {
+      "epoch": 0.05434153832130224,
+      "grad_norm": 1.1554538011550903,
+      "learning_rate": 4.452610552959183e-05,
+      "loss": 0.6168,
+      "step": 7750
+    },
+    {
+      "epoch": 0.054558904474587444,
+      "grad_norm": 1.1653157472610474,
+      "learning_rate": 4.447355047201428e-05,
+      "loss": 0.615,
+      "step": 7781
+    },
+    {
+      "epoch": 0.05477627062787265,
+      "grad_norm": 1.0511231422424316,
+      "learning_rate": 4.4420775658066414e-05,
+      "loss": 0.6215,
+      "step": 7812
+    },
+    {
+      "epoch": 0.05499363678115786,
+      "grad_norm": 1.3468266725540161,
+      "learning_rate": 4.436778168330484e-05,
+      "loss": 0.6221,
+      "step": 7843
+    },
+    {
+      "epoch": 0.05521100293444307,
+      "grad_norm": 1.3238797187805176,
+      "learning_rate": 4.4314569145759353e-05,
+      "loss": 0.611,
+      "step": 7874
+    },
+    {
+      "epoch": 0.05542836908772828,
+      "grad_norm": 1.1107763051986694,
+      "learning_rate": 4.42611386459262e-05,
+      "loss": 0.6119,
+      "step": 7905
+    },
+    {
+      "epoch": 0.05564573524101349,
+      "grad_norm": 1.3667259216308594,
+      "learning_rate": 4.420749078676133e-05,
+      "loss": 0.6134,
+      "step": 7936
+    },
+    {
+      "epoch": 0.055863101394298696,
+      "grad_norm": 1.0336949825286865,
+      "learning_rate": 4.4153626173673516e-05,
+      "loss": 0.613,
+      "step": 7967
+    },
+    {
+      "epoch": 0.056080467547583904,
+      "grad_norm": 1.0911316871643066,
+      "learning_rate": 4.409954541451762e-05,
+      "loss": 0.6107,
+      "step": 7998
+    },
+    {
+      "epoch": 0.05629783370086911,
+      "grad_norm": 1.2516382932662964,
+      "learning_rate": 4.404524911958764e-05,
+      "loss": 0.6138,
+      "step": 8029
+    },
+    {
+      "epoch": 0.056515199854154326,
+      "grad_norm": 1.1678277254104614,
+      "learning_rate": 4.399073790160989e-05,
+      "loss": 0.6162,
+      "step": 8060
+    },
+    {
+      "epoch": 0.05673256600743953,
+      "grad_norm": 1.4072996377944946,
+      "learning_rate": 4.393601237573607e-05,
+      "loss": 0.614,
+      "step": 8091
+    },
+    {
+      "epoch": 0.05694993216072474,
+      "grad_norm": 1.2312837839126587,
+      "learning_rate": 4.388107315953628e-05,
+      "loss": 0.6203,
+      "step": 8122
+    },
+    {
+      "epoch": 0.05716729831400995,
+      "grad_norm": 1.1720649003982544,
+      "learning_rate": 4.382592087299212e-05,
+      "loss": 0.619,
+      "step": 8153
+    },
+    {
+      "epoch": 0.057384664467295156,
+      "grad_norm": 1.0711950063705444,
+      "learning_rate": 4.377055613848964e-05,
+      "loss": 0.6124,
+      "step": 8184
+    },
+    {
+      "epoch": 0.05760203062058037,
+      "grad_norm": 1.1664263010025024,
+      "learning_rate": 4.3714979580812355e-05,
+      "loss": 0.6073,
+      "step": 8215
+    },
+    {
+      "epoch": 0.05781939677386558,
+      "grad_norm": 0.9937831163406372,
+      "learning_rate": 4.365919182713416e-05,
+      "loss": 0.6101,
+      "step": 8246
+    },
+    {
+      "epoch": 0.058036762927150785,
+      "grad_norm": 1.0545841455459595,
+      "learning_rate": 4.360319350701226e-05,
+      "loss": 0.6089,
+      "step": 8277
+    },
+    {
+      "epoch": 0.05825412908043599,
+      "grad_norm": 1.120007038116455,
+      "learning_rate": 4.3546985252380115e-05,
+      "loss": 0.6078,
+      "step": 8308
+    },
+    {
+      "epoch": 0.0584714952337212,
+      "grad_norm": 1.1087831258773804,
+      "learning_rate": 4.349056769754021e-05,
+      "loss": 0.601,
+      "step": 8339
+    },
+    {
+      "epoch": 0.058688861387006415,
+      "grad_norm": 1.1370675563812256,
+      "learning_rate": 4.3433941479156994e-05,
+      "loss": 0.6103,
+      "step": 8370
+    },
+    {
+      "epoch": 0.05890622754029162,
+      "grad_norm": 1.182654857635498,
+      "learning_rate": 4.3377107236249647e-05,
+      "loss": 0.6115,
+      "step": 8401
+    },
+    {
+      "epoch": 0.05912359369357683,
+      "grad_norm": 1.2606432437896729,
+      "learning_rate": 4.332006561018488e-05,
+      "loss": 0.6139,
+      "step": 8432
+    },
+    {
+      "epoch": 0.05934095984686204,
+      "grad_norm": 1.1618047952651978,
+      "learning_rate": 4.3262817244669683e-05,
+      "loss": 0.6053,
+      "step": 8463
+    },
+    {
+      "epoch": 0.059558326000147245,
+      "grad_norm": 1.1281821727752686,
+      "learning_rate": 4.3205362785744083e-05,
+      "loss": 0.6084,
+      "step": 8494
+    },
+    {
+      "epoch": 0.05977569215343246,
+      "grad_norm": 1.15248703956604,
+      "learning_rate": 4.314770288177384e-05,
+      "loss": 0.6128,
+      "step": 8525
+    },
+    {
+      "epoch": 0.05999305830671767,
+      "grad_norm": 1.2170960903167725,
+      "learning_rate": 4.308983818344313e-05,
+      "loss": 0.6134,
+      "step": 8556
+    },
+    {
+      "epoch": 0.060210424460002875,
+      "grad_norm": 1.085260033607483,
+      "learning_rate": 4.3031769343747206e-05,
+      "loss": 0.6212,
+      "step": 8587
+    },
+    {
+      "epoch": 0.06042779061328808,
+      "grad_norm": 1.2243624925613403,
+      "learning_rate": 4.297349701798505e-05,
+      "loss": 0.605,
+      "step": 8618
+    },
+    {
+      "epoch": 0.06064515676657329,
+      "grad_norm": 1.124769687652588,
+      "learning_rate": 4.2915021863751916e-05,
+      "loss": 0.6074,
+      "step": 8649
+    },
+    {
+      "epoch": 0.060862522919858504,
+      "grad_norm": 1.893545150756836,
+      "learning_rate": 4.285634454093198e-05,
+      "loss": 0.6055,
+      "step": 8680
+    },
+    {
+      "epoch": 0.06107988907314371,
+      "grad_norm": 1.2186870574951172,
+      "learning_rate": 4.279746571169086e-05,
+      "loss": 0.6121,
+      "step": 8711
+    },
+    {
+      "epoch": 0.06129725522642892,
+      "grad_norm": 1.0006957054138184,
+      "learning_rate": 4.2738386040468136e-05,
+      "loss": 0.6079,
+      "step": 8742
+    },
+    {
+      "epoch": 0.06151462137971413,
+      "grad_norm": 1.0913995504379272,
+      "learning_rate": 4.2679106193969866e-05,
+      "loss": 0.6107,
+      "step": 8773
+    },
+    {
+      "epoch": 0.061731987532999334,
+      "grad_norm": 1.4647611379623413,
+      "learning_rate": 4.261962684116106e-05,
+      "loss": 0.6188,
+      "step": 8804
+    },
+    {
+      "epoch": 0.06194935368628455,
+      "grad_norm": 1.0805052518844604,
+      "learning_rate": 4.2559948653258145e-05,
+      "loss": 0.6137,
+      "step": 8835
+    },
+    {
+      "epoch": 0.062166719839569756,
+      "grad_norm": 1.0512675046920776,
+      "learning_rate": 4.250007230372134e-05,
+      "loss": 0.6052,
+      "step": 8866
+    },
+    {
+      "epoch": 0.062384085992854964,
+      "grad_norm": 1.1758863925933838,
+      "learning_rate": 4.2439998468247126e-05,
+      "loss": 0.6148,
+      "step": 8897
+    },
+    {
+      "epoch": 0.06260145214614017,
+      "grad_norm": 1.0526351928710938,
+      "learning_rate": 4.2379727824760566e-05,
+      "loss": 0.5973,
+      "step": 8928
+    },
+    {
+      "epoch": 0.06281881829942539,
+      "grad_norm": 1.077563762664795,
+      "learning_rate": 4.231926105340768e-05,
+      "loss": 0.6114,
+      "step": 8959
+    },
+    {
+      "epoch": 0.06303618445271059,
+      "grad_norm": 1.0116938352584839,
+      "learning_rate": 4.225859883654776e-05,
+      "loss": 0.6146,
+      "step": 8990
+    },
+    {
+      "epoch": 0.0632535506059958,
+      "grad_norm": 1.1967271566390991,
+      "learning_rate": 4.219774185874569e-05,
+      "loss": 0.5972,
+      "step": 9021
+    },
+    {
+      "epoch": 0.06347091675928102,
+      "grad_norm": 1.1610004901885986,
+      "learning_rate": 4.213669080676418e-05,
+      "loss": 0.6081,
+      "step": 9052
+    },
+    {
+      "epoch": 0.06368828291256622,
+      "grad_norm": 1.0253360271453857,
+      "learning_rate": 4.2075446369556056e-05,
+      "loss": 0.6118,
+      "step": 9083
+    },
+    {
+      "epoch": 0.06390564906585143,
+      "grad_norm": 1.140599012374878,
+      "learning_rate": 4.201400923825648e-05,
+      "loss": 0.6091,
+      "step": 9114
+    },
+    {
+      "epoch": 0.06412301521913663,
+      "grad_norm": 1.0821545124053955,
+      "learning_rate": 4.195238010617511e-05,
+      "loss": 0.6028,
+      "step": 9145
+    },
+    {
+      "epoch": 0.06434038137242185,
+      "grad_norm": 1.2161924839019775,
+      "learning_rate": 4.1890559668788344e-05,
+      "loss": 0.6113,
+      "step": 9176
+    },
+    {
+      "epoch": 0.06455774752570706,
+      "grad_norm": 1.128986120223999,
+      "learning_rate": 4.1828548623731405e-05,
+      "loss": 0.6125,
+      "step": 9207
+    },
+    {
+      "epoch": 0.06477511367899226,
+      "grad_norm": 1.0566622018814087,
+      "learning_rate": 4.1766347670790506e-05,
+      "loss": 0.6055,
+      "step": 9238
+    },
+    {
+      "epoch": 0.06499247983227747,
+      "grad_norm": 1.0849272012710571,
+      "learning_rate": 4.170395751189495e-05,
+      "loss": 0.6059,
+      "step": 9269
+    },
+    {
+      "epoch": 0.06520984598556268,
+      "grad_norm": 1.2344416379928589,
+      "learning_rate": 4.164137885110921e-05,
+      "loss": 0.6081,
+      "step": 9300
+    },
+    {
+      "epoch": 0.06542721213884789,
+      "grad_norm": 1.0132557153701782,
+      "learning_rate": 4.157861239462495e-05,
+      "loss": 0.6103,
+      "step": 9331
+    },
+    {
+      "epoch": 0.0656445782921331,
+      "grad_norm": 1.0176981687545776,
+      "learning_rate": 4.1515658850753114e-05,
+      "loss": 0.6086,
+      "step": 9362
+    },
+    {
+      "epoch": 0.0658619444454183,
+      "grad_norm": 1.2429553270339966,
+      "learning_rate": 4.145251892991588e-05,
+      "loss": 0.6091,
+      "step": 9393
+    },
+    {
+      "epoch": 0.06607931059870352,
+      "grad_norm": 1.1481480598449707,
+      "learning_rate": 4.138919334463868e-05,
+      "loss": 0.6048,
+      "step": 9424
+    },
+    {
+      "epoch": 0.06629667675198872,
+      "grad_norm": 0.9623891711235046,
+      "learning_rate": 4.1325682809542124e-05,
+      "loss": 0.5995,
+      "step": 9455
+    },
+    {
+      "epoch": 0.06651404290527393,
+      "grad_norm": 1.2031786441802979,
+      "learning_rate": 4.126198804133398e-05,
+      "loss": 0.6008,
+      "step": 9486
+    },
+    {
+      "epoch": 0.06673140905855915,
+      "grad_norm": 1.3027530908584595,
+      "learning_rate": 4.1198109758801055e-05,
+      "loss": 0.5993,
+      "step": 9517
+    },
+    {
+      "epoch": 0.06694877521184435,
+      "grad_norm": 1.1712346076965332,
+      "learning_rate": 4.113404868280107e-05,
+      "loss": 0.5996,
+      "step": 9548
+    },
+    {
+      "epoch": 0.06716614136512956,
+      "grad_norm": 1.1397625207901,
+      "learning_rate": 4.106980553625457e-05,
+      "loss": 0.5979,
+      "step": 9579
+    },
+    {
+      "epoch": 0.06738350751841476,
+      "grad_norm": 1.122983694076538,
+      "learning_rate": 4.100538104413674e-05,
+      "loss": 0.599,
+      "step": 9610
+    },
+    {
+      "epoch": 0.06760087367169998,
+      "grad_norm": 1.0782618522644043,
+      "learning_rate": 4.09407759334692e-05,
+      "loss": 0.6005,
+      "step": 9641
+    },
+    {
+      "epoch": 0.0678182398249852,
+      "grad_norm": 1.0563392639160156,
+      "learning_rate": 4.087599093331186e-05,
+      "loss": 0.6105,
+      "step": 9672
+    },
+    {
+      "epoch": 0.0680356059782704,
+      "grad_norm": 1.0684717893600464,
+      "learning_rate": 4.081102677475462e-05,
+      "loss": 0.6092,
+      "step": 9703
+    },
+    {
+      "epoch": 0.06825297213155561,
+      "grad_norm": 1.152811884880066,
+      "learning_rate": 4.0745884190909194e-05,
+      "loss": 0.6035,
+      "step": 9734
+    },
+    {
+      "epoch": 0.06847033828484081,
+      "grad_norm": 1.1413112878799438,
+      "learning_rate": 4.0680563916900796e-05,
+      "loss": 0.6018,
+      "step": 9765
+    },
+    {
+      "epoch": 0.06868770443812602,
+      "grad_norm": 0.981604814529419,
+      "learning_rate": 4.0615066689859815e-05,
+      "loss": 0.6035,
+      "step": 9796
+    },
+    {
+      "epoch": 0.06890507059141124,
+      "grad_norm": 1.0713250637054443,
+      "learning_rate": 4.0549393248913584e-05,
+      "loss": 0.597,
+      "step": 9827
+    },
+    {
+      "epoch": 0.06912243674469644,
+      "grad_norm": 0.949456512928009,
+      "learning_rate": 4.048354433517794e-05,
+      "loss": 0.6063,
+      "step": 9858
+    },
+    {
+      "epoch": 0.06933980289798165,
+      "grad_norm": 1.0702073574066162,
+      "learning_rate": 4.0417520691748916e-05,
+      "loss": 0.5963,
+      "step": 9889
+    },
+    {
+      "epoch": 0.06955716905126685,
+      "grad_norm": 0.9974942803382874,
+      "learning_rate": 4.035132306369438e-05,
+      "loss": 0.611,
+      "step": 9920
+    },
+    {
+      "epoch": 0.06977453520455207,
+      "grad_norm": 1.0833642482757568,
+      "learning_rate": 4.028495219804555e-05,
+      "loss": 0.6014,
+      "step": 9951
+    },
+    {
+      "epoch": 0.06999190135783728,
+      "grad_norm": 1.181322693824768,
+      "learning_rate": 4.021840884378864e-05,
+      "loss": 0.599,
+      "step": 9982
+    },
+    {
+      "epoch": 0.07020926751112248,
+      "grad_norm": 1.0088189840316772,
+      "learning_rate": 4.015169375185633e-05,
+      "loss": 0.5955,
+      "step": 10013
+    },
+    {
+      "epoch": 0.0704266336644077,
+      "grad_norm": 1.0920222997665405,
+      "learning_rate": 4.0084807675119396e-05,
+      "loss": 0.5985,
+      "step": 10044
+    },
+    {
+      "epoch": 0.0706439998176929,
+      "grad_norm": 1.0450975894927979,
+      "learning_rate": 4.0017751368378106e-05,
+      "loss": 0.6069,
+      "step": 10075
+    },
+    {
+      "epoch": 0.07086136597097811,
+      "grad_norm": 1.1613439321517944,
+      "learning_rate": 3.995052558835377e-05,
+      "loss": 0.5992,
+      "step": 10106
+    },
+    {
+      "epoch": 0.07107873212426333,
+      "grad_norm": 1.2720811367034912,
+      "learning_rate": 3.988313109368017e-05,
+      "loss": 0.6012,
+      "step": 10137
+    },
+    {
+      "epoch": 0.07129609827754853,
+      "grad_norm": 1.0315334796905518,
+      "learning_rate": 3.981556864489504e-05,
+      "loss": 0.6149,
+      "step": 10168
+    },
+    {
+      "epoch": 0.07151346443083374,
+      "grad_norm": 1.3320891857147217,
+      "learning_rate": 3.974783900443142e-05,
+      "loss": 0.6013,
+      "step": 10199
+    },
+    {
+      "epoch": 0.07173083058411894,
+      "grad_norm": 1.7277195453643799,
+      "learning_rate": 3.9679942936609095e-05,
+      "loss": 0.5983,
+      "step": 10230
+    },
+    {
+      "epoch": 0.07194819673740416,
+      "grad_norm": 1.1056753396987915,
+      "learning_rate": 3.961188120762596e-05,
+      "loss": 0.5952,
+      "step": 10261
+    },
+    {
+      "epoch": 0.07216556289068937,
+      "grad_norm": 1.3461558818817139,
+      "learning_rate": 3.954365458554938e-05,
+      "loss": 0.5975,
+      "step": 10292
+    },
+    {
+      "epoch": 0.07238292904397457,
+      "grad_norm": 1.1405422687530518,
+      "learning_rate": 3.947526384030751e-05,
+      "loss": 0.602,
+      "step": 10323
+    },
+    {
+      "epoch": 0.07260029519725979,
+      "grad_norm": 1.2912482023239136,
+      "learning_rate": 3.9406709743680624e-05,
+      "loss": 0.6014,
+      "step": 10354
+    },
+    {
+      "epoch": 0.07281766135054499,
+      "grad_norm": 1.1548583507537842,
+      "learning_rate": 3.9337993069292366e-05,
+      "loss": 0.6053,
+      "step": 10385
+    },
+    {
+      "epoch": 0.0730350275038302,
+      "grad_norm": 1.0558302402496338,
+      "learning_rate": 3.926911459260109e-05,
+      "loss": 0.6062,
+      "step": 10416
+    },
+    {
+      "epoch": 0.07325239365711542,
+      "grad_norm": 1.1658024787902832,
+      "learning_rate": 3.920007509089102e-05,
+      "loss": 0.6016,
+      "step": 10447
+    },
+    {
+      "epoch": 0.07346975981040062,
+      "grad_norm": 1.1093552112579346,
+      "learning_rate": 3.913087534326357e-05,
+      "loss": 0.6025,
+      "step": 10478
+    },
+    {
+      "epoch": 0.07368712596368583,
+      "grad_norm": 1.0908467769622803,
+      "learning_rate": 3.9061516130628475e-05,
+      "loss": 0.5954,
+      "step": 10509
+    },
+    {
+      "epoch": 0.07390449211697103,
+      "grad_norm": 1.0856796503067017,
+      "learning_rate": 3.8991998235695025e-05,
+      "loss": 0.5983,
+      "step": 10540
+    },
+    {
+      "epoch": 0.07412185827025625,
+      "grad_norm": 1.0330291986465454,
+      "learning_rate": 3.8922322442963224e-05,
+      "loss": 0.5955,
+      "step": 10571
+    },
+    {
+      "epoch": 0.07433922442354146,
+      "grad_norm": 0.9688258171081543,
+      "learning_rate": 3.885248953871491e-05,
+      "loss": 0.6001,
+      "step": 10602
+    },
+    {
+      "epoch": 0.07455659057682666,
+      "grad_norm": 1.1808383464813232,
+      "learning_rate": 3.8782500311004915e-05,
+      "loss": 0.5996,
+      "step": 10633
+    },
+    {
+      "epoch": 0.07477395673011188,
+      "grad_norm": 1.1267834901809692,
+      "learning_rate": 3.871235554965218e-05,
+      "loss": 0.5962,
+      "step": 10664
+    },
+    {
+      "epoch": 0.07499132288339708,
+      "grad_norm": 1.1001136302947998,
+      "learning_rate": 3.864205604623078e-05,
+      "loss": 0.6031,
+      "step": 10695
+    },
+    {
+      "epoch": 0.07520868903668229,
+      "grad_norm": 1.009621024131775,
+      "learning_rate": 3.857160259406107e-05,
+      "loss": 0.5976,
+      "step": 10726
+    },
+    {
+      "epoch": 0.0754260551899675,
+      "grad_norm": 0.933811366558075,
+      "learning_rate": 3.8500995988200674e-05,
+      "loss": 0.604,
+      "step": 10757
+    },
+    {
+      "epoch": 0.0756434213432527,
+      "grad_norm": 1.2187680006027222,
+      "learning_rate": 3.843023702543556e-05,
+      "loss": 0.5983,
+      "step": 10788
+    },
+    {
+      "epoch": 0.07586078749653792,
+      "grad_norm": 1.0286976099014282,
+      "learning_rate": 3.8359326504270984e-05,
+      "loss": 0.596,
+      "step": 10819
+    },
+    {
+      "epoch": 0.07607815364982312,
+      "grad_norm": 1.0752719640731812,
+      "learning_rate": 3.828826522492255e-05,
+      "loss": 0.596,
+      "step": 10850
+    },
+    {
+      "epoch": 0.07629551980310834,
+      "grad_norm": 1.1018482446670532,
+      "learning_rate": 3.821705398930713e-05,
+      "loss": 0.603,
+      "step": 10881
+    },
+    {
+      "epoch": 0.07651288595639355,
+      "grad_norm": 0.980965793132782,
+      "learning_rate": 3.814569360103385e-05,
+      "loss": 0.6021,
+      "step": 10912
+    },
+    {
+      "epoch": 0.07673025210967875,
+      "grad_norm": 0.9588106870651245,
+      "learning_rate": 3.807418486539499e-05,
+      "loss": 0.6017,
+      "step": 10943
+    },
+    {
+      "epoch": 0.07694761826296397,
+      "grad_norm": 1.2048044204711914,
+      "learning_rate": 3.80025285893569e-05,
+      "loss": 0.6049,
+      "step": 10974
+    },
+    {
+      "epoch": 0.07716498441624917,
+      "grad_norm": 1.0373165607452393,
+      "learning_rate": 3.793072558155093e-05,
+      "loss": 0.5914,
+      "step": 11005
+    },
+    {
+      "epoch": 0.07738235056953438,
+      "grad_norm": 1.0112730264663696,
+      "learning_rate": 3.785877665226426e-05,
+      "loss": 0.5949,
+      "step": 11036
+    },
+    {
+      "epoch": 0.0775997167228196,
+      "grad_norm": 1.0219500064849854,
+      "learning_rate": 3.778668261343079e-05,
+      "loss": 0.5918,
+      "step": 11067
+    },
+    {
+      "epoch": 0.0778170828761048,
+      "grad_norm": 1.1104828119277954,
+      "learning_rate": 3.771444427862192e-05,
+      "loss": 0.6029,
+      "step": 11098
+    },
+    {
+      "epoch": 0.07803444902939001,
+      "grad_norm": 0.9852742552757263,
+      "learning_rate": 3.7642062463037465e-05,
+      "loss": 0.5839,
+      "step": 11129
+    },
+    {
+      "epoch": 0.07825181518267521,
+      "grad_norm": 1.0593205690383911,
+      "learning_rate": 3.7569537983496373e-05,
+      "loss": 0.5966,
+      "step": 11160
+    },
+    {
+      "epoch": 0.07846918133596043,
+      "grad_norm": 1.2858383655548096,
+      "learning_rate": 3.749687165842753e-05,
+      "loss": 0.5939,
+      "step": 11191
+    },
+    {
+      "epoch": 0.07868654748924564,
+      "grad_norm": 1.2006022930145264,
+      "learning_rate": 3.7424064307860536e-05,
+      "loss": 0.5965,
+      "step": 11222
+    },
+    {
+      "epoch": 0.07890391364253084,
+      "grad_norm": 0.9741569757461548,
+      "learning_rate": 3.735111675341645e-05,
+      "loss": 0.5971,
+      "step": 11253
+    },
+    {
+      "epoch": 0.07912127979581605,
+      "grad_norm": 1.0724074840545654,
+      "learning_rate": 3.7278029818298524e-05,
+      "loss": 0.5901,
+      "step": 11284
+    },
+    {
+      "epoch": 0.07933864594910126,
+      "grad_norm": 1.102207064628601,
+      "learning_rate": 3.720480432728287e-05,
+      "loss": 0.602,
+      "step": 11315
+    },
+    {
+      "epoch": 0.07955601210238647,
+      "grad_norm": 1.0976966619491577,
+      "learning_rate": 3.71314411067092e-05,
+      "loss": 0.6002,
+      "step": 11346
+    },
+    {
+      "epoch": 0.07977337825567168,
+      "grad_norm": 1.0534948110580444,
+      "learning_rate": 3.70579409844715e-05,
+      "loss": 0.5801,
+      "step": 11377
+    },
+    {
+      "epoch": 0.07999074440895688,
+      "grad_norm": 1.1666125059127808,
+      "learning_rate": 3.698430479000865e-05,
+      "loss": 0.5992,
+      "step": 11408
+    },
+    {
+      "epoch": 0.0802081105622421,
+      "grad_norm": 1.0316112041473389,
+      "learning_rate": 3.691053335429509e-05,
+      "loss": 0.5942,
+      "step": 11439
+    },
+    {
+      "epoch": 0.0804254767155273,
+      "grad_norm": 1.0085538625717163,
+      "learning_rate": 3.683662750983147e-05,
+      "loss": 0.5978,
+      "step": 11470
+    },
+    {
+      "epoch": 0.08064284286881251,
+      "grad_norm": 1.0875978469848633,
+      "learning_rate": 3.676258809063518e-05,
+      "loss": 0.5949,
+      "step": 11501
+    },
+    {
+      "epoch": 0.08086020902209773,
+      "grad_norm": 1.1905012130737305,
+      "learning_rate": 3.6688415932231004e-05,
+      "loss": 0.6018,
+      "step": 11532
+    },
+    {
+      "epoch": 0.08107757517538293,
+      "grad_norm": 1.1717733144760132,
+      "learning_rate": 3.661411187164166e-05,
+      "loss": 0.5986,
+      "step": 11563
+    },
+    {
+      "epoch": 0.08129494132866814,
+      "grad_norm": 1.053768515586853,
+      "learning_rate": 3.65396767473784e-05,
+      "loss": 0.5831,
+      "step": 11594
+    },
+    {
+      "epoch": 0.08151230748195334,
+      "grad_norm": 1.095425009727478,
+      "learning_rate": 3.6465111399431465e-05,
+      "loss": 0.5967,
+      "step": 11625
+    },
+    {
+      "epoch": 0.08172967363523856,
+      "grad_norm": 1.005953311920166,
+      "learning_rate": 3.6390416669260674e-05,
+      "loss": 0.5834,
+      "step": 11656
+    },
+    {
+      "epoch": 0.08194703978852377,
+      "grad_norm": 1.0991450548171997,
+      "learning_rate": 3.63155933997859e-05,
+      "loss": 0.5951,
+      "step": 11687
+    },
+    {
+      "epoch": 0.08216440594180897,
+      "grad_norm": 1.1069071292877197,
+      "learning_rate": 3.624064243537758e-05,
+      "loss": 0.5989,
+      "step": 11718
+    },
+    {
+      "epoch": 0.08238177209509419,
+      "grad_norm": 1.099542498588562,
+      "learning_rate": 3.616556462184716e-05,
+      "loss": 0.5949,
+      "step": 11749
+    },
+    {
+      "epoch": 0.08259913824837939,
+      "grad_norm": 1.0950508117675781,
+      "learning_rate": 3.609036080643755e-05,
+      "loss": 0.5911,
+      "step": 11780
+    },
+    {
+      "epoch": 0.0828165044016646,
+      "grad_norm": 0.9489970803260803,
+      "learning_rate": 3.60150318378136e-05,
+      "loss": 0.5922,
+      "step": 11811
+    },
+    {
+      "epoch": 0.08303387055494982,
+      "grad_norm": 0.9877674579620361,
+      "learning_rate": 3.5939578566052465e-05,
+      "loss": 0.5927,
+      "step": 11842
+    },
+    {
+      "epoch": 0.08325123670823502,
+      "grad_norm": 1.060177206993103,
+      "learning_rate": 3.586400184263408e-05,
+      "loss": 0.5899,
+      "step": 11873
+    },
+    {
+      "epoch": 0.08346860286152023,
+      "grad_norm": 1.0531491041183472,
+      "learning_rate": 3.578830252043148e-05,
+      "loss": 0.6004,
+      "step": 11904
+    },
+    {
+      "epoch": 0.08368596901480543,
+      "grad_norm": 0.9791742563247681,
+      "learning_rate": 3.571248145370125e-05,
+      "loss": 0.5926,
+      "step": 11935
+    },
+    {
+      "epoch": 0.08390333516809065,
+      "grad_norm": 1.0492030382156372,
+      "learning_rate": 3.5636539498073794e-05,
+      "loss": 0.5914,
+      "step": 11966
+    },
+    {
+      "epoch": 0.08412070132137586,
+      "grad_norm": 1.2035881280899048,
+      "learning_rate": 3.556047751054378e-05,
+      "loss": 0.589,
+      "step": 11997
+    },
+    {
+      "epoch": 0.08433806747466106,
+      "grad_norm": 1.0384137630462646,
+      "learning_rate": 3.548429634946039e-05,
+      "loss": 0.5894,
+      "step": 12028
+    },
+    {
+      "epoch": 0.08455543362794628,
+      "grad_norm": 1.1555323600769043,
+      "learning_rate": 3.540799687451768e-05,
+      "loss": 0.5899,
+      "step": 12059
+    },
+    {
+      "epoch": 0.08477279978123148,
+      "grad_norm": 1.2114530801773071,
+      "learning_rate": 3.533157994674485e-05,
+      "loss": 0.5888,
+      "step": 12090
+    },
+    {
+      "epoch": 0.08499016593451669,
+      "grad_norm": 1.145213007926941,
+      "learning_rate": 3.5255046428496546e-05,
+      "loss": 0.5887,
+      "step": 12121
+    },
+    {
+      "epoch": 0.08520753208780191,
+      "grad_norm": 1.0313454866409302,
+      "learning_rate": 3.517839718344311e-05,
+      "loss": 0.598,
+      "step": 12152
+    },
+    {
+      "epoch": 0.08542489824108711,
+      "grad_norm": 1.018702745437622,
+      "learning_rate": 3.510163307656086e-05,
+      "loss": 0.5931,
+      "step": 12183
+    },
+    {
+      "epoch": 0.08564226439437232,
+      "grad_norm": 1.0365880727767944,
+      "learning_rate": 3.5024754974122324e-05,
+      "loss": 0.5954,
+      "step": 12214
+    },
+    {
+      "epoch": 0.08585963054765752,
+      "grad_norm": 1.0658811330795288,
+      "learning_rate": 3.494776374368643e-05,
+      "loss": 0.5953,
+      "step": 12245
+    },
+    {
+      "epoch": 0.08607699670094274,
+      "grad_norm": 1.046661615371704,
+      "learning_rate": 3.4870660254088724e-05,
+      "loss": 0.5786,
+      "step": 12276
+    },
+    {
+      "epoch": 0.08629436285422795,
+      "grad_norm": 1.0220295190811157,
+      "learning_rate": 3.479344537543164e-05,
+      "loss": 0.5879,
+      "step": 12307
+    },
+    {
+      "epoch": 0.08651172900751315,
+      "grad_norm": 1.0286898612976074,
+      "learning_rate": 3.4716119979074565e-05,
+      "loss": 0.5882,
+      "step": 12338
+    },
+    {
+      "epoch": 0.08672909516079837,
+      "grad_norm": 1.060483694076538,
+      "learning_rate": 3.463868493762412e-05,
+      "loss": 0.5887,
+      "step": 12369
+    },
+    {
+      "epoch": 0.08694646131408357,
+      "grad_norm": 1.264788269996643,
+      "learning_rate": 3.456114112492418e-05,
+      "loss": 0.5895,
+      "step": 12400
+    },
+    {
+      "epoch": 0.08716382746736878,
+      "grad_norm": 0.9751125574111938,
+      "learning_rate": 3.4483489416046164e-05,
+      "loss": 0.5846,
+      "step": 12431
+    },
+    {
+      "epoch": 0.087381193620654,
+      "grad_norm": 1.1198588609695435,
+      "learning_rate": 3.440573068727905e-05,
+      "loss": 0.5878,
+      "step": 12462
+    },
+    {
+      "epoch": 0.0875985597739392,
+      "grad_norm": 1.0425771474838257,
+      "learning_rate": 3.4327865816119495e-05,
+      "loss": 0.5903,
+      "step": 12493
+    },
+    {
+      "epoch": 0.08781592592722441,
+      "grad_norm": 0.9820422530174255,
+      "learning_rate": 3.4249895681262025e-05,
+      "loss": 0.5885,
+      "step": 12524
+    },
+    {
+      "epoch": 0.08803329208050961,
+      "grad_norm": 1.0234076976776123,
+      "learning_rate": 3.417182116258899e-05,
+      "loss": 0.5967,
+      "step": 12555
+    },
+    {
+      "epoch": 0.08825065823379483,
+      "grad_norm": 0.9766228795051575,
+      "learning_rate": 3.409364314116074e-05,
+      "loss": 0.5923,
+      "step": 12586
+    },
+    {
+      "epoch": 0.08846802438708004,
+      "grad_norm": 0.9033060073852539,
+      "learning_rate": 3.401536249920559e-05,
+      "loss": 0.5892,
+      "step": 12617
+    },
+    {
+      "epoch": 0.08868539054036524,
+      "grad_norm": 1.1264934539794922,
+      "learning_rate": 3.393698012010998e-05,
+      "loss": 0.5917,
+      "step": 12648
+    },
+    {
+      "epoch": 0.08890275669365046,
+      "grad_norm": 1.0052556991577148,
+      "learning_rate": 3.385849688840839e-05,
+      "loss": 0.5885,
+      "step": 12679
+    },
+    {
+      "epoch": 0.08912012284693566,
+      "grad_norm": 1.0888159275054932,
+      "learning_rate": 3.3779913689773414e-05,
+      "loss": 0.5885,
+      "step": 12710
+    },
+    {
+      "epoch": 0.08933748900022087,
+      "grad_norm": 1.0458447933197021,
+      "learning_rate": 3.370123141100578e-05,
+      "loss": 0.5923,
+      "step": 12741
+    },
+    {
+      "epoch": 0.08955485515350609,
+      "grad_norm": 1.055434226989746,
+      "learning_rate": 3.3622450940024305e-05,
+      "loss": 0.5889,
+      "step": 12772
+    },
+    {
+      "epoch": 0.08977222130679129,
+      "grad_norm": 1.000533103942871,
+      "learning_rate": 3.35435731658559e-05,
+      "loss": 0.5833,
+      "step": 12803
+    },
+    {
+      "epoch": 0.0899895874600765,
+      "grad_norm": 1.0112768411636353,
+      "learning_rate": 3.346459897862552e-05,
+      "loss": 0.5886,
+      "step": 12834
+    },
+    {
+      "epoch": 0.0902069536133617,
+      "grad_norm": 0.9883559942245483,
+      "learning_rate": 3.338552926954613e-05,
+      "loss": 0.5886,
+      "step": 12865
+    },
+    {
+      "epoch": 0.09042431976664692,
+      "grad_norm": 1.0933095216751099,
+      "learning_rate": 3.330636493090868e-05,
+      "loss": 0.5873,
+      "step": 12896
+    },
+    {
+      "epoch": 0.09064168591993213,
+      "grad_norm": 1.1089582443237305,
+      "learning_rate": 3.322710685607193e-05,
+      "loss": 0.5845,
+      "step": 12927
+    },
+    {
+      "epoch": 0.09085905207321733,
+      "grad_norm": 1.0311237573623657,
+      "learning_rate": 3.314775593945251e-05,
+      "loss": 0.588,
+      "step": 12958
+    },
+    {
+      "epoch": 0.09107641822650255,
+      "grad_norm": 1.0495847463607788,
+      "learning_rate": 3.3068313076514714e-05,
+      "loss": 0.5839,
+      "step": 12989
+    },
+    {
+      "epoch": 0.09129378437978775,
+      "grad_norm": 1.046921730041504,
+      "learning_rate": 3.298877916376047e-05,
+      "loss": 0.5877,
+      "step": 13020
+    },
+    {
+      "epoch": 0.09151115053307296,
+      "grad_norm": 0.9927343130111694,
+      "learning_rate": 3.290915509871915e-05,
+      "loss": 0.5898,
+      "step": 13051
+    },
+    {
+      "epoch": 0.09172851668635817,
+      "grad_norm": 0.9319870471954346,
+      "learning_rate": 3.282944177993753e-05,
+      "loss": 0.5886,
+      "step": 13082
+    },
+    {
+      "epoch": 0.09194588283964338,
+      "grad_norm": 1.0408544540405273,
+      "learning_rate": 3.274964010696957e-05,
+      "loss": 0.5946,
+      "step": 13113
+    },
+    {
+      "epoch": 0.09216324899292859,
+      "grad_norm": 1.0984251499176025,
+      "learning_rate": 3.266975098036629e-05,
+      "loss": 0.5813,
+      "step": 13144
+    },
+    {
+      "epoch": 0.09238061514621379,
+      "grad_norm": 1.0376449823379517,
+      "learning_rate": 3.258977530166562e-05,
+      "loss": 0.5855,
+      "step": 13175
+    },
+    {
+      "epoch": 0.092597981299499,
+      "grad_norm": 0.9627104997634888,
+      "learning_rate": 3.250971397338227e-05,
+      "loss": 0.5846,
+      "step": 13206
+    },
+    {
+      "epoch": 0.09281534745278422,
+      "grad_norm": 1.2040653228759766,
+      "learning_rate": 3.2429567898997404e-05,
+      "loss": 0.5872,
+      "step": 13237
+    },
+    {
+      "epoch": 0.09303271360606942,
+      "grad_norm": 1.0091586112976074,
+      "learning_rate": 3.234933798294859e-05,
+      "loss": 0.5911,
+      "step": 13268
+    },
+    {
+      "epoch": 0.09325007975935463,
+      "grad_norm": 1.0941787958145142,
+      "learning_rate": 3.2269025130619535e-05,
+      "loss": 0.5844,
+      "step": 13299
+    },
+    {
+      "epoch": 0.09346744591263983,
+      "grad_norm": 0.9720754027366638,
+      "learning_rate": 3.218863024832985e-05,
+      "loss": 0.5898,
+      "step": 13330
+    },
+    {
+      "epoch": 0.09368481206592505,
+      "grad_norm": 0.8847430944442749,
+      "learning_rate": 3.2108154243324864e-05,
+      "loss": 0.5804,
+      "step": 13361
+    },
+    {
+      "epoch": 0.09390217821921026,
+      "grad_norm": 1.0070865154266357,
+      "learning_rate": 3.2027598023765345e-05,
+      "loss": 0.5916,
+      "step": 13392
+    },
+    {
+      "epoch": 0.09411954437249546,
+      "grad_norm": 0.9538979530334473,
+      "learning_rate": 3.194696249871729e-05,
+      "loss": 0.5984,
+      "step": 13423
+    },
+    {
+      "epoch": 0.09433691052578068,
+      "grad_norm": 1.0414469242095947,
+      "learning_rate": 3.186624857814164e-05,
+      "loss": 0.5828,
+      "step": 13454
+    },
+    {
+      "epoch": 0.0945542766790659,
+      "grad_norm": 1.0353257656097412,
+      "learning_rate": 3.178545717288401e-05,
+      "loss": 0.5835,
+      "step": 13485
+    },
+    {
+      "epoch": 0.0947716428323511,
+      "grad_norm": 1.0454679727554321,
+      "learning_rate": 3.170458919466444e-05,
+      "loss": 0.5909,
+      "step": 13516
+    },
+    {
+      "epoch": 0.09498900898563631,
+      "grad_norm": 1.246871829032898,
+      "learning_rate": 3.1623645556067063e-05,
+      "loss": 0.5829,
+      "step": 13547
+    },
+    {
+      "epoch": 0.09520637513892151,
+      "grad_norm": 1.0331645011901855,
+      "learning_rate": 3.154262717052985e-05,
+      "loss": 0.5919,
+      "step": 13578
+    },
+    {
+      "epoch": 0.09542374129220672,
+      "grad_norm": 0.8984120488166809,
+      "learning_rate": 3.146153495233426e-05,
+      "loss": 0.5853,
+      "step": 13609
+    },
+    {
+      "epoch": 0.09564110744549194,
+      "grad_norm": 1.0582929849624634,
+      "learning_rate": 3.1380369816594944e-05,
+      "loss": 0.5828,
+      "step": 13640
+    },
+    {
+      "epoch": 0.09585847359877714,
+      "grad_norm": 1.151041865348816,
+      "learning_rate": 3.129913267924946e-05,
+      "loss": 0.582,
+      "step": 13671
+    },
+    {
+      "epoch": 0.09607583975206235,
+      "grad_norm": 0.9678122401237488,
+      "learning_rate": 3.121782445704782e-05,
+      "loss": 0.5865,
+      "step": 13702
+    },
+    {
+      "epoch": 0.09629320590534755,
+      "grad_norm": 1.033832311630249,
+      "learning_rate": 3.11364460675423e-05,
+      "loss": 0.5797,
+      "step": 13733
+    },
+    {
+      "epoch": 0.09651057205863277,
+      "grad_norm": 0.8917058706283569,
+      "learning_rate": 3.1054998429076934e-05,
+      "loss": 0.5829,
+      "step": 13764
+    },
+    {
+      "epoch": 0.09672793821191798,
+      "grad_norm": 0.9396641850471497,
+      "learning_rate": 3.097348246077728e-05,
+      "loss": 0.5825,
+      "step": 13795
+    },
+    {
+      "epoch": 0.09694530436520318,
+      "grad_norm": 1.0871998071670532,
+      "learning_rate": 3.0891899082539924e-05,
+      "loss": 0.5851,
+      "step": 13826
+    },
+    {
+      "epoch": 0.0971626705184884,
+      "grad_norm": 1.1403570175170898,
+      "learning_rate": 3.0810249215022233e-05,
+      "loss": 0.5807,
+      "step": 13857
+    },
+    {
+      "epoch": 0.0973800366717736,
+      "grad_norm": 1.0246673822402954,
+      "learning_rate": 3.0728533779631865e-05,
+      "loss": 0.5795,
+      "step": 13888
+    },
+    {
+      "epoch": 0.09759740282505881,
+      "grad_norm": 0.9431213140487671,
+      "learning_rate": 3.064675369851637e-05,
+      "loss": 0.5825,
+      "step": 13919
+    },
+    {
+      "epoch": 0.09781476897834403,
+      "grad_norm": 1.1028188467025757,
+      "learning_rate": 3.056490989455289e-05,
+      "loss": 0.5885,
+      "step": 13950
+    },
+    {
+      "epoch": 0.09803213513162923,
+      "grad_norm": 1.125482201576233,
+      "learning_rate": 3.0483003291337596e-05,
+      "loss": 0.5849,
+      "step": 13981
+    },
+    {
+      "epoch": 0.09824950128491444,
+      "grad_norm": 0.9831985831260681,
+      "learning_rate": 3.040103481317539e-05,
+      "loss": 0.586,
+      "step": 14012
+    },
+    {
+      "epoch": 0.09846686743819964,
+      "grad_norm": 1.0038244724273682,
+      "learning_rate": 3.03190053850694e-05,
+      "loss": 0.5894,
+      "step": 14043
+    },
+    {
+      "epoch": 0.09868423359148486,
+      "grad_norm": 0.8766770362854004,
+      "learning_rate": 3.0236915932710573e-05,
+      "loss": 0.5773,
+      "step": 14074
+    },
+    {
+      "epoch": 0.09890159974477007,
+      "grad_norm": 0.9358529448509216,
+      "learning_rate": 3.0154767382467232e-05,
+      "loss": 0.5887,
+      "step": 14105
+    },
+    {
+      "epoch": 0.09911896589805527,
+      "grad_norm": 0.8988749384880066,
+      "learning_rate": 3.0072560661374582e-05,
+      "loss": 0.5815,
+      "step": 14136
+    },
+    {
+      "epoch": 0.09933633205134049,
+      "grad_norm": 1.08226478099823,
+      "learning_rate": 2.999029669712431e-05,
+      "loss": 0.5944,
+      "step": 14167
+    },
+    {
+      "epoch": 0.09955369820462569,
+      "grad_norm": 1.046980619430542,
+      "learning_rate": 2.990797641805408e-05,
+      "loss": 0.5824,
+      "step": 14198
+    },
+    {
+      "epoch": 0.0997710643579109,
+      "grad_norm": 0.96205073595047,
+      "learning_rate": 2.982560075313704e-05,
+      "loss": 0.5847,
+      "step": 14229
+    },
+    {
+      "epoch": 0.09998843051119612,
+      "grad_norm": 0.9914370775222778,
+      "learning_rate": 2.9743170631971368e-05,
+      "loss": 0.5824,
+      "step": 14260
+    },
+    {
+      "epoch": 0.10020579666448132,
+      "grad_norm": 0.922265887260437,
+      "learning_rate": 2.9660686984769792e-05,
+      "loss": 0.5907,
+      "step": 14291
+    },
+    {
+      "epoch": 0.10042316281776653,
+      "grad_norm": 0.971159815788269,
+      "learning_rate": 2.9578150742349047e-05,
+      "loss": 0.5806,
+      "step": 14322
+    },
+    {
+      "epoch": 0.10064052897105173,
+      "grad_norm": 1.1327799558639526,
+      "learning_rate": 2.949556283611942e-05,
+      "loss": 0.5812,
+      "step": 14353
+    },
+    {
+      "epoch": 0.10085789512433695,
+      "grad_norm": 1.1632689237594604,
+      "learning_rate": 2.9412924198074206e-05,
+      "loss": 0.5813,
+      "step": 14384
+    },
+    {
+      "epoch": 0.10107526127762216,
+      "grad_norm": 0.9936057329177856,
+      "learning_rate": 2.9330235760779208e-05,
+      "loss": 0.5803,
+      "step": 14415
+    },
+    {
+      "epoch": 0.10129262743090736,
+      "grad_norm": 1.1123740673065186,
+      "learning_rate": 2.9247498457362188e-05,
+      "loss": 0.5767,
+      "step": 14446
+    },
+    {
+      "epoch": 0.10150999358419258,
+      "grad_norm": 0.908643364906311,
+      "learning_rate": 2.9164713221502373e-05,
+      "loss": 0.5797,
+      "step": 14477
+    },
+    {
+      "epoch": 0.10172735973747778,
+      "grad_norm": 0.8761305809020996,
+      "learning_rate": 2.9081880987419912e-05,
+      "loss": 0.589,
+      "step": 14508
+    },
+    {
+      "epoch": 0.10194472589076299,
+      "grad_norm": 1.0904477834701538,
+      "learning_rate": 2.8999002689865296e-05,
+      "loss": 0.5796,
+      "step": 14539
+    },
+    {
+      "epoch": 0.1021620920440482,
+      "grad_norm": 1.060160756111145,
+      "learning_rate": 2.8916079264108852e-05,
+      "loss": 0.5837,
+      "step": 14570
+    },
+    {
+      "epoch": 0.1023794581973334,
+      "grad_norm": 1.0514498949050903,
+      "learning_rate": 2.883311164593017e-05,
+      "loss": 0.5792,
+      "step": 14601
+    },
+    {
+      "epoch": 0.10259682435061862,
+      "grad_norm": 1.0152157545089722,
+      "learning_rate": 2.875010077160754e-05,
+      "loss": 0.574,
+      "step": 14632
+    },
+    {
+      "epoch": 0.10281419050390382,
+      "grad_norm": 0.974791944026947,
+      "learning_rate": 2.866704757790741e-05,
+      "loss": 0.5819,
+      "step": 14663
+    },
+    {
+      "epoch": 0.10303155665718904,
+      "grad_norm": 0.9271855354309082,
+      "learning_rate": 2.858395300207376e-05,
+      "loss": 0.5798,
+      "step": 14694
+    },
+    {
+      "epoch": 0.10324892281047425,
+      "grad_norm": 0.9933458566665649,
+      "learning_rate": 2.8500817981817607e-05,
+      "loss": 0.5796,
+      "step": 14725
+    },
+    {
+      "epoch": 0.10346628896375945,
+      "grad_norm": 0.9245687127113342,
+      "learning_rate": 2.8417643455306336e-05,
+      "loss": 0.5771,
+      "step": 14756
+    },
+    {
+      "epoch": 0.10368365511704467,
+      "grad_norm": 0.9483000040054321,
+      "learning_rate": 2.8334430361153185e-05,
+      "loss": 0.5825,
+      "step": 14787
+    },
+    {
+      "epoch": 0.10390102127032987,
+      "grad_norm": 0.9054533839225769,
+      "learning_rate": 2.8251179638406612e-05,
+      "loss": 0.5817,
+      "step": 14818
+    },
+    {
+      "epoch": 0.10411838742361508,
+      "grad_norm": 1.0200084447860718,
+      "learning_rate": 2.8167892226539704e-05,
+      "loss": 0.5788,
+      "step": 14849
+    },
+    {
+      "epoch": 0.1043357535769003,
+      "grad_norm": 0.9364084005355835,
+      "learning_rate": 2.8084569065439588e-05,
+      "loss": 0.5796,
+      "step": 14880
+    },
+    {
+      "epoch": 0.1045531197301855,
+      "grad_norm": 0.9658653736114502,
+      "learning_rate": 2.8001211095396807e-05,
+      "loss": 0.5825,
+      "step": 14911
+    },
+    {
+      "epoch": 0.10477048588347071,
+      "grad_norm": 0.8612831234931946,
+      "learning_rate": 2.791781925709473e-05,
+      "loss": 0.5842,
+      "step": 14942
+    },
+    {
+      "epoch": 0.10498785203675591,
+      "grad_norm": 0.9673991799354553,
+      "learning_rate": 2.7834394491598908e-05,
+      "loss": 0.5776,
+      "step": 14973
+    },
+    {
+      "epoch": 0.10520521819004112,
+      "grad_norm": 1.092144250869751,
+      "learning_rate": 2.7750937740346485e-05,
+      "loss": 0.5785,
+      "step": 15004
+    },
+    {
+      "epoch": 0.10542258434332634,
+      "grad_norm": 1.0162559747695923,
+      "learning_rate": 2.7667449945135564e-05,
+      "loss": 0.5821,
+      "step": 15035
+    },
+    {
+      "epoch": 0.10563995049661154,
+      "grad_norm": 0.9578939080238342,
+      "learning_rate": 2.7583932048114557e-05,
+      "loss": 0.5781,
+      "step": 15066
+    },
+    {
+      "epoch": 0.10585731664989675,
+      "grad_norm": 0.9934547543525696,
+      "learning_rate": 2.7500384991771587e-05,
+      "loss": 0.5826,
+      "step": 15097
+    },
+    {
+      "epoch": 0.10607468280318195,
+      "grad_norm": 0.9542795419692993,
+      "learning_rate": 2.7416809718923825e-05,
+      "loss": 0.583,
+      "step": 15128
+    },
+    {
+      "epoch": 0.10629204895646717,
+      "grad_norm": 0.9436337947845459,
+      "learning_rate": 2.7333207172706864e-05,
+      "loss": 0.5832,
+      "step": 15159
+    },
+    {
+      "epoch": 0.10650941510975238,
+      "grad_norm": 0.9200606942176819,
+      "learning_rate": 2.7249578296564088e-05,
+      "loss": 0.5766,
+      "step": 15190
+    },
+    {
+      "epoch": 0.10672678126303758,
+      "grad_norm": 1.0009740591049194,
+      "learning_rate": 2.7165924034235973e-05,
+      "loss": 0.5835,
+      "step": 15221
+    },
+    {
+      "epoch": 0.1069441474163228,
+      "grad_norm": 0.9597145915031433,
+      "learning_rate": 2.708224532974953e-05,
+      "loss": 0.5728,
+      "step": 15252
+    }
+  ],
+  "logging_steps": 31,
+  "max_steps": 30517,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 3052,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.1258280394232955e+19,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-15260/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff6a84b6c2a69c365aedcd04f176d62d0993a049ac50b28b1fc7fee54f4267a8
+size 5304

checkpoint-18312/config.json

@@ -0,0 +1,39 @@
+{
+  "_name_or_path": "meta-llama/Llama-3.1-8B-Instruct",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.0.dev0",
+  "use_cache": true,
+  "vocab_size": 128256
+}

checkpoint-18312/generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.44.0.dev0"
+}

checkpoint-18312/model-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8586746032ca2a712be9adeb302f11125101dd345d36bea9bb4d92265761297b
+size 4886466168

checkpoint-18312/model-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23fa5aaf8ac56c17c821980c9aee44fcbc2a617fc570e875e6fdb93d92886c72
+size 4832007448

checkpoint-18312/model-00003-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a223e87240e50dfb05634ea909cf1ec715d37a12b681b14a3a6e97f28688f2b
+size 4999813112

checkpoint-18312/model-00004-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3047d199c611c6d12b1cca780f033ad89df9c076f742674f7ac4a892ca9573b3
+size 4999813128

checkpoint-18312/model-00005-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a2f91d749d664ecd22c71f3d54488966fc6899a03924b8fea3613c8f83398a8
+size 4832007496

checkpoint-18312/model-00006-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9cbccce82be060349a3666ef22a62bb3f1fb4ee73d1523e8d20bfd2ee6d8e85
+size 4999813120

checkpoint-18312/model-00007-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d00be9546497a5a6726ba09faaf4c510b3b053bda4ace055d3bf3e8c28f945cf
+size 2571158184

checkpoint-18312/model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 32121044992
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00007-of-00007.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.norm.weight": "model-00007-of-00007.safetensors"
+  }
+}

checkpoint-18312/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54b10c84d9a3ee7718da1e24de45ffdd81fdaa663bce8438623561ac18149314
+size 15385036334

checkpoint-18312/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
+size 14244

checkpoint-18312/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0e65c3d6f29e706fd941a38280ce5628189a6998eac6d29abbeab00ad838d00
+size 1064

checkpoint-18312/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff6a84b6c2a69c365aedcd04f176d62d0993a049ac50b28b1fc7fee54f4267a8
+size 5304

checkpoint-21364/config.json

@@ -0,0 +1,39 @@
+{
+  "_name_or_path": "meta-llama/Llama-3.1-8B-Instruct",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.0.dev0",
+  "use_cache": true,
+  "vocab_size": 128256
+}

checkpoint-21364/generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.44.0.dev0"
+}

checkpoint-21364/model-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37b1b9af313cbf3bba45242b7bd452818f6129154bab8f6aeaf72b930b9a830b
+size 4886466168

checkpoint-21364/model-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23fa5aaf8ac56c17c821980c9aee44fcbc2a617fc570e875e6fdb93d92886c72
+size 4832007448

checkpoint-21364/model-00003-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a223e87240e50dfb05634ea909cf1ec715d37a12b681b14a3a6e97f28688f2b
+size 4999813112