Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

checkpoint-15260/config.json

@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "meta-llama/Llama-3.1-8B",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.47.0.dev0",
+  "use_cache": true,
+  "vocab_size": 128256
+}

checkpoint-15260/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.47.0.dev0"
+}

checkpoint-15260/model-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f079bf02c172bedd74770e82510a16cdb11e3bf4fcb319716f17266a072444c
+size 4886466168

checkpoint-15260/model-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0901d2e156c6ec7a375c3e90ae5183970af0445d3dc0e53c031b45673775dd64
+size 4832007448

checkpoint-15260/model-00003-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afa0e22bcade9527b5aeb86066e139cc30979f8492c753ff3277e6189863ab97
+size 4999813112

checkpoint-15260/model-00004-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc4f5352f30ea036b0a4e24ba9b029dd94a2b30a716c81acc1c4a5e7bb47a042
+size 4999813128

checkpoint-15260/model-00005-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d9450b209df4cb026b6669e618fcdb6ff4f5bd0a711fb2e502afe0b95eb3ad7
+size 4832007496

checkpoint-15260/model-00006-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3e1e9e71428df158a722a220767f0e0da8c1734019054bb6fab096ef1125b33
+size 4999813120

checkpoint-15260/model-00007-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3834470fcd8c571e64d7418e131d13d5c3097d28c44306f2c681c76e11dbc0e9
+size 2571158184

checkpoint-15260/model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 32121044992
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00007-of-00007.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.norm.weight": "model-00007-of-00007.safetensors"
+  }
+}

checkpoint-15260/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f3977110a166ba1eabe0c3c79942af4d58fdff092701de4ff9806e7572af35d
+size 15385036334

checkpoint-15260/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:386fcc8cc1089aade9450d86fb239ea3483f455fd2d78d8378645feecfec9d69
+size 14244

checkpoint-15260/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ce335347efaebf038b8915bc0a8d2d587a4a1aa08ad0e015b4bc7cc4fba634e
+size 1064

checkpoint-15260/trainer_state.json

@@ -0,0 +1,3477 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.11189631625741285,
+  "eval_steps": 500,
+  "global_step": 15260,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.00022731230694494092,
+      "grad_norm": 3.798166513442993,
+      "learning_rate": 1.0157273918741808e-06,
+      "loss": 0.7918,
+      "step": 31
+    },
+    {
+      "epoch": 0.00045462461388988184,
+      "grad_norm": 3.6598377227783203,
+      "learning_rate": 2.0314547837483616e-06,
+      "loss": 0.7597,
+      "step": 62
+    },
+    {
+      "epoch": 0.0006819369208348227,
+      "grad_norm": 3.45603084564209,
+      "learning_rate": 3.0471821756225426e-06,
+      "loss": 0.7381,
+      "step": 93
+    },
+    {
+      "epoch": 0.0009092492277797637,
+      "grad_norm": 3.278182029724121,
+      "learning_rate": 4.062909567496723e-06,
+      "loss": 0.7406,
+      "step": 124
+    },
+    {
+      "epoch": 0.0011365615347247045,
+      "grad_norm": 3.872504949569702,
+      "learning_rate": 5.078636959370905e-06,
+      "loss": 0.7342,
+      "step": 155
+    },
+    {
+      "epoch": 0.0013638738416696455,
+      "grad_norm": 3.2023379802703857,
+      "learning_rate": 6.094364351245085e-06,
+      "loss": 0.7267,
+      "step": 186
+    },
+    {
+      "epoch": 0.0015911861486145864,
+      "grad_norm": 3.1996734142303467,
+      "learning_rate": 7.110091743119267e-06,
+      "loss": 0.716,
+      "step": 217
+    },
+    {
+      "epoch": 0.0018184984555595274,
+      "grad_norm": 3.4770455360412598,
+      "learning_rate": 8.125819134993446e-06,
+      "loss": 0.724,
+      "step": 248
+    },
+    {
+      "epoch": 0.0020458107625044683,
+      "grad_norm": 3.2189555168151855,
+      "learning_rate": 9.141546526867629e-06,
+      "loss": 0.7239,
+      "step": 279
+    },
+    {
+      "epoch": 0.002273123069449409,
+      "grad_norm": 3.2377490997314453,
+      "learning_rate": 1.015727391874181e-05,
+      "loss": 0.7052,
+      "step": 310
+    },
+    {
+      "epoch": 0.0025004353763943502,
+      "grad_norm": 3.038553476333618,
+      "learning_rate": 1.117300131061599e-05,
+      "loss": 0.7158,
+      "step": 341
+    },
+    {
+      "epoch": 0.002727747683339291,
+      "grad_norm": 2.802546739578247,
+      "learning_rate": 1.218872870249017e-05,
+      "loss": 0.7106,
+      "step": 372
+    },
+    {
+      "epoch": 0.002955059990284232,
+      "grad_norm": 3.919581174850464,
+      "learning_rate": 1.3204456094364351e-05,
+      "loss": 0.7082,
+      "step": 403
+    },
+    {
+      "epoch": 0.003182372297229173,
+      "grad_norm": 3.072122097015381,
+      "learning_rate": 1.4220183486238533e-05,
+      "loss": 0.7074,
+      "step": 434
+    },
+    {
+      "epoch": 0.003409684604174114,
+      "grad_norm": 2.744469404220581,
+      "learning_rate": 1.5235910878112714e-05,
+      "loss": 0.7174,
+      "step": 465
+    },
+    {
+      "epoch": 0.0036369969111190547,
+      "grad_norm": 2.8211464881896973,
+      "learning_rate": 1.6251638269986893e-05,
+      "loss": 0.7075,
+      "step": 496
+    },
+    {
+      "epoch": 0.003864309218063996,
+      "grad_norm": 2.747333288192749,
+      "learning_rate": 1.7267365661861077e-05,
+      "loss": 0.7132,
+      "step": 527
+    },
+    {
+      "epoch": 0.004091621525008937,
+      "grad_norm": 2.681431531906128,
+      "learning_rate": 1.8283093053735257e-05,
+      "loss": 0.7098,
+      "step": 558
+    },
+    {
+      "epoch": 0.004318933831953878,
+      "grad_norm": 2.6222891807556152,
+      "learning_rate": 1.9298820445609438e-05,
+      "loss": 0.7071,
+      "step": 589
+    },
+    {
+      "epoch": 0.004546246138898818,
+      "grad_norm": 2.5538346767425537,
+      "learning_rate": 2.031454783748362e-05,
+      "loss": 0.7054,
+      "step": 620
+    },
+    {
+      "epoch": 0.004773558445843759,
+      "grad_norm": 2.253528594970703,
+      "learning_rate": 2.13302752293578e-05,
+      "loss": 0.7025,
+      "step": 651
+    },
+    {
+      "epoch": 0.0050008707527887004,
+      "grad_norm": 2.6388654708862305,
+      "learning_rate": 2.234600262123198e-05,
+      "loss": 0.696,
+      "step": 682
+    },
+    {
+      "epoch": 0.005228183059733642,
+      "grad_norm": 2.2375001907348633,
+      "learning_rate": 2.336173001310616e-05,
+      "loss": 0.7122,
+      "step": 713
+    },
+    {
+      "epoch": 0.005455495366678582,
+      "grad_norm": 2.626136064529419,
+      "learning_rate": 2.437745740498034e-05,
+      "loss": 0.7038,
+      "step": 744
+    },
+    {
+      "epoch": 0.005682807673623523,
+      "grad_norm": 2.080169439315796,
+      "learning_rate": 2.5393184796854525e-05,
+      "loss": 0.6889,
+      "step": 775
+    },
+    {
+      "epoch": 0.005910119980568464,
+      "grad_norm": 2.415144443511963,
+      "learning_rate": 2.6408912188728702e-05,
+      "loss": 0.6967,
+      "step": 806
+    },
+    {
+      "epoch": 0.006137432287513405,
+      "grad_norm": 2.165386915206909,
+      "learning_rate": 2.7424639580602886e-05,
+      "loss": 0.695,
+      "step": 837
+    },
+    {
+      "epoch": 0.006364744594458346,
+      "grad_norm": 2.08313250541687,
+      "learning_rate": 2.8440366972477066e-05,
+      "loss": 0.696,
+      "step": 868
+    },
+    {
+      "epoch": 0.006592056901403287,
+      "grad_norm": 2.293937921524048,
+      "learning_rate": 2.9456094364351244e-05,
+      "loss": 0.6944,
+      "step": 899
+    },
+    {
+      "epoch": 0.006819369208348228,
+      "grad_norm": 2.081148386001587,
+      "learning_rate": 3.0471821756225428e-05,
+      "loss": 0.6827,
+      "step": 930
+    },
+    {
+      "epoch": 0.007046681515293168,
+      "grad_norm": 1.9917105436325073,
+      "learning_rate": 3.148754914809961e-05,
+      "loss": 0.6964,
+      "step": 961
+    },
+    {
+      "epoch": 0.0072739938222381095,
+      "grad_norm": 1.9086660146713257,
+      "learning_rate": 3.2503276539973785e-05,
+      "loss": 0.6948,
+      "step": 992
+    },
+    {
+      "epoch": 0.007501306129183051,
+      "grad_norm": 2.0621390342712402,
+      "learning_rate": 3.351900393184797e-05,
+      "loss": 0.689,
+      "step": 1023
+    },
+    {
+      "epoch": 0.007728618436127992,
+      "grad_norm": 2.267564296722412,
+      "learning_rate": 3.453473132372215e-05,
+      "loss": 0.6939,
+      "step": 1054
+    },
+    {
+      "epoch": 0.007955930743072933,
+      "grad_norm": 2.179840087890625,
+      "learning_rate": 3.555045871559633e-05,
+      "loss": 0.6905,
+      "step": 1085
+    },
+    {
+      "epoch": 0.008183243050017873,
+      "grad_norm": 2.2900586128234863,
+      "learning_rate": 3.6566186107470514e-05,
+      "loss": 0.6885,
+      "step": 1116
+    },
+    {
+      "epoch": 0.008410555356962814,
+      "grad_norm": 1.993735671043396,
+      "learning_rate": 3.7581913499344695e-05,
+      "loss": 0.6931,
+      "step": 1147
+    },
+    {
+      "epoch": 0.008637867663907756,
+      "grad_norm": 2.3171041011810303,
+      "learning_rate": 3.8597640891218876e-05,
+      "loss": 0.703,
+      "step": 1178
+    },
+    {
+      "epoch": 0.008865179970852696,
+      "grad_norm": 2.094472885131836,
+      "learning_rate": 3.9613368283093056e-05,
+      "loss": 0.6872,
+      "step": 1209
+    },
+    {
+      "epoch": 0.009092492277797636,
+      "grad_norm": 2.0125269889831543,
+      "learning_rate": 4.062909567496724e-05,
+      "loss": 0.6855,
+      "step": 1240
+    },
+    {
+      "epoch": 0.009319804584742578,
+      "grad_norm": 1.6527774333953857,
+      "learning_rate": 4.164482306684142e-05,
+      "loss": 0.6736,
+      "step": 1271
+    },
+    {
+      "epoch": 0.009547116891687519,
+      "grad_norm": 2.0705461502075195,
+      "learning_rate": 4.26605504587156e-05,
+      "loss": 0.6838,
+      "step": 1302
+    },
+    {
+      "epoch": 0.00977442919863246,
+      "grad_norm": 1.7904468774795532,
+      "learning_rate": 4.367627785058978e-05,
+      "loss": 0.6835,
+      "step": 1333
+    },
+    {
+      "epoch": 0.010001741505577401,
+      "grad_norm": 2.62326979637146,
+      "learning_rate": 4.469200524246396e-05,
+      "loss": 0.6853,
+      "step": 1364
+    },
+    {
+      "epoch": 0.010229053812522341,
+      "grad_norm": 1.9389066696166992,
+      "learning_rate": 4.570773263433814e-05,
+      "loss": 0.6841,
+      "step": 1395
+    },
+    {
+      "epoch": 0.010456366119467283,
+      "grad_norm": 1.8236297369003296,
+      "learning_rate": 4.672346002621232e-05,
+      "loss": 0.6723,
+      "step": 1426
+    },
+    {
+      "epoch": 0.010683678426412224,
+      "grad_norm": 4.768039703369141,
+      "learning_rate": 4.77391874180865e-05,
+      "loss": 0.6903,
+      "step": 1457
+    },
+    {
+      "epoch": 0.010910990733357164,
+      "grad_norm": 1.7986795902252197,
+      "learning_rate": 4.875491480996068e-05,
+      "loss": 0.678,
+      "step": 1488
+    },
+    {
+      "epoch": 0.011138303040302106,
+      "grad_norm": 2.8209362030029297,
+      "learning_rate": 4.977064220183487e-05,
+      "loss": 0.6819,
+      "step": 1519
+    },
+    {
+      "epoch": 0.011365615347247046,
+      "grad_norm": 1.7705190181732178,
+      "learning_rate": 4.9999915451558777e-05,
+      "loss": 0.6849,
+      "step": 1550
+    },
+    {
+      "epoch": 0.011592927654191986,
+      "grad_norm": 2.0149953365325928,
+      "learning_rate": 4.999955597496219e-05,
+      "loss": 0.6776,
+      "step": 1581
+    },
+    {
+      "epoch": 0.011820239961136928,
+      "grad_norm": 52.01716232299805,
+      "learning_rate": 4.9998914381774255e-05,
+      "loss": 0.6813,
+      "step": 1612
+    },
+    {
+      "epoch": 0.012047552268081869,
+      "grad_norm": 1.6921148300170898,
+      "learning_rate": 4.999799067923527e-05,
+      "loss": 0.6895,
+      "step": 1643
+    },
+    {
+      "epoch": 0.01227486457502681,
+      "grad_norm": 1.831264853477478,
+      "learning_rate": 4.999678487776908e-05,
+      "loss": 0.6686,
+      "step": 1674
+    },
+    {
+      "epoch": 0.012502176881971751,
+      "grad_norm": 1.7291995286941528,
+      "learning_rate": 4.9995296990983006e-05,
+      "loss": 0.6743,
+      "step": 1705
+    },
+    {
+      "epoch": 0.012729489188916691,
+      "grad_norm": 1.9407203197479248,
+      "learning_rate": 4.999352703566763e-05,
+      "loss": 0.6681,
+      "step": 1736
+    },
+    {
+      "epoch": 0.012956801495861633,
+      "grad_norm": 1.414100170135498,
+      "learning_rate": 4.999147503179668e-05,
+      "loss": 0.6645,
+      "step": 1767
+    },
+    {
+      "epoch": 0.013184113802806574,
+      "grad_norm": 1.9148963689804077,
+      "learning_rate": 4.998914100252672e-05,
+      "loss": 0.6695,
+      "step": 1798
+    },
+    {
+      "epoch": 0.013411426109751514,
+      "grad_norm": 1.8338842391967773,
+      "learning_rate": 4.998652497419696e-05,
+      "loss": 0.6618,
+      "step": 1829
+    },
+    {
+      "epoch": 0.013638738416696456,
+      "grad_norm": 1.627996802330017,
+      "learning_rate": 4.9983626976328927e-05,
+      "loss": 0.6624,
+      "step": 1860
+    },
+    {
+      "epoch": 0.013866050723641396,
+      "grad_norm": 1.810975432395935,
+      "learning_rate": 4.998044704162613e-05,
+      "loss": 0.6672,
+      "step": 1891
+    },
+    {
+      "epoch": 0.014093363030586337,
+      "grad_norm": 1.9406816959381104,
+      "learning_rate": 4.9976985205973705e-05,
+      "loss": 0.6609,
+      "step": 1922
+    },
+    {
+      "epoch": 0.014320675337531279,
+      "grad_norm": 1.5875732898712158,
+      "learning_rate": 4.997324150843799e-05,
+      "loss": 0.6713,
+      "step": 1953
+    },
+    {
+      "epoch": 0.014547987644476219,
+      "grad_norm": 1.7407218217849731,
+      "learning_rate": 4.99692159912661e-05,
+      "loss": 0.6668,
+      "step": 1984
+    },
+    {
+      "epoch": 0.014775299951421161,
+      "grad_norm": 1.7074840068817139,
+      "learning_rate": 4.996490869988546e-05,
+      "loss": 0.6562,
+      "step": 2015
+    },
+    {
+      "epoch": 0.015002612258366101,
+      "grad_norm": 1.4265679121017456,
+      "learning_rate": 4.996031968290326e-05,
+      "loss": 0.6674,
+      "step": 2046
+    },
+    {
+      "epoch": 0.015229924565311042,
+      "grad_norm": 1.645156979560852,
+      "learning_rate": 4.995544899210594e-05,
+      "loss": 0.6606,
+      "step": 2077
+    },
+    {
+      "epoch": 0.015457236872255984,
+      "grad_norm": 1.3875256776809692,
+      "learning_rate": 4.9950296682458583e-05,
+      "loss": 0.6795,
+      "step": 2108
+    },
+    {
+      "epoch": 0.015684549179200924,
+      "grad_norm": 1.5201404094696045,
+      "learning_rate": 4.994486281210429e-05,
+      "loss": 0.6533,
+      "step": 2139
+    },
+    {
+      "epoch": 0.015911861486145866,
+      "grad_norm": 1.9790576696395874,
+      "learning_rate": 4.9939147442363566e-05,
+      "loss": 0.6518,
+      "step": 2170
+    },
+    {
+      "epoch": 0.016139173793090805,
+      "grad_norm": 1.691832423210144,
+      "learning_rate": 4.9933150637733574e-05,
+      "loss": 0.6579,
+      "step": 2201
+    },
+    {
+      "epoch": 0.016366486100035747,
+      "grad_norm": 1.9516123533248901,
+      "learning_rate": 4.992687246588743e-05,
+      "loss": 0.6673,
+      "step": 2232
+    },
+    {
+      "epoch": 0.01659379840698069,
+      "grad_norm": 1.5482674837112427,
+      "learning_rate": 4.992031299767347e-05,
+      "loss": 0.6482,
+      "step": 2263
+    },
+    {
+      "epoch": 0.016821110713925627,
+      "grad_norm": 1.5505927801132202,
+      "learning_rate": 4.9913472307114386e-05,
+      "loss": 0.6527,
+      "step": 2294
+    },
+    {
+      "epoch": 0.01704842302087057,
+      "grad_norm": 1.3475325107574463,
+      "learning_rate": 4.9906350471406446e-05,
+      "loss": 0.6493,
+      "step": 2325
+    },
+    {
+      "epoch": 0.01727573532781551,
+      "grad_norm": 1.4781843423843384,
+      "learning_rate": 4.989894757091861e-05,
+      "loss": 0.6455,
+      "step": 2356
+    },
+    {
+      "epoch": 0.01750304763476045,
+      "grad_norm": 3.436141014099121,
+      "learning_rate": 4.989126368919158e-05,
+      "loss": 0.6541,
+      "step": 2387
+    },
+    {
+      "epoch": 0.017730359941705392,
+      "grad_norm": 1.505778193473816,
+      "learning_rate": 4.988329891293693e-05,
+      "loss": 0.6688,
+      "step": 2418
+    },
+    {
+      "epoch": 0.017957672248650334,
+      "grad_norm": 1.482499122619629,
+      "learning_rate": 4.987505333203608e-05,
+      "loss": 0.6457,
+      "step": 2449
+    },
+    {
+      "epoch": 0.018184984555595272,
+      "grad_norm": 1.6124916076660156,
+      "learning_rate": 4.9866527039539276e-05,
+      "loss": 0.665,
+      "step": 2480
+    },
+    {
+      "epoch": 0.018412296862540214,
+      "grad_norm": 1.3330082893371582,
+      "learning_rate": 4.9857720131664594e-05,
+      "loss": 0.6414,
+      "step": 2511
+    },
+    {
+      "epoch": 0.018639609169485157,
+      "grad_norm": 1.6367557048797607,
+      "learning_rate": 4.9848632707796773e-05,
+      "loss": 0.6352,
+      "step": 2542
+    },
+    {
+      "epoch": 0.0188669214764301,
+      "grad_norm": 1.2797504663467407,
+      "learning_rate": 4.9839264870486155e-05,
+      "loss": 0.6548,
+      "step": 2573
+    },
+    {
+      "epoch": 0.019094233783375037,
+      "grad_norm": 1.4322324991226196,
+      "learning_rate": 4.9829616725447526e-05,
+      "loss": 0.6499,
+      "step": 2604
+    },
+    {
+      "epoch": 0.01932154609031998,
+      "grad_norm": 1.4474719762802124,
+      "learning_rate": 4.981968838155888e-05,
+      "loss": 0.6362,
+      "step": 2635
+    },
+    {
+      "epoch": 0.01954885839726492,
+      "grad_norm": 1.577005386352539,
+      "learning_rate": 4.980947995086024e-05,
+      "loss": 0.6417,
+      "step": 2666
+    },
+    {
+      "epoch": 0.01977617070420986,
+      "grad_norm": 1.3451883792877197,
+      "learning_rate": 4.979899154855234e-05,
+      "loss": 0.6554,
+      "step": 2697
+    },
+    {
+      "epoch": 0.020003483011154802,
+      "grad_norm": 1.3591116666793823,
+      "learning_rate": 4.9788223292995386e-05,
+      "loss": 0.645,
+      "step": 2728
+    },
+    {
+      "epoch": 0.020230795318099744,
+      "grad_norm": 1.4452534914016724,
+      "learning_rate": 4.977717530570768e-05,
+      "loss": 0.648,
+      "step": 2759
+    },
+    {
+      "epoch": 0.020458107625044682,
+      "grad_norm": 1.2939035892486572,
+      "learning_rate": 4.976584771136425e-05,
+      "loss": 0.6442,
+      "step": 2790
+    },
+    {
+      "epoch": 0.020685419931989624,
+      "grad_norm": 1.4697262048721313,
+      "learning_rate": 4.975424063779547e-05,
+      "loss": 0.6412,
+      "step": 2821
+    },
+    {
+      "epoch": 0.020912732238934566,
+      "grad_norm": 1.4037450551986694,
+      "learning_rate": 4.974235421598557e-05,
+      "loss": 0.6468,
+      "step": 2852
+    },
+    {
+      "epoch": 0.021140044545879505,
+      "grad_norm": 1.2538812160491943,
+      "learning_rate": 4.973018858007122e-05,
+      "loss": 0.6321,
+      "step": 2883
+    },
+    {
+      "epoch": 0.021367356852824447,
+      "grad_norm": 1.2294617891311646,
+      "learning_rate": 4.9717743867339963e-05,
+      "loss": 0.6342,
+      "step": 2914
+    },
+    {
+      "epoch": 0.02159466915976939,
+      "grad_norm": 1.5091283321380615,
+      "learning_rate": 4.9705020218228695e-05,
+      "loss": 0.6366,
+      "step": 2945
+    },
+    {
+      "epoch": 0.021821981466714328,
+      "grad_norm": 1.1950831413269043,
+      "learning_rate": 4.969201777632205e-05,
+      "loss": 0.6314,
+      "step": 2976
+    },
+    {
+      "epoch": 0.02204929377365927,
+      "grad_norm": 1.3976637125015259,
+      "learning_rate": 4.9678736688350846e-05,
+      "loss": 0.6362,
+      "step": 3007
+    },
+    {
+      "epoch": 0.02227660608060421,
+      "grad_norm": 1.2981493473052979,
+      "learning_rate": 4.966517710419033e-05,
+      "loss": 0.6374,
+      "step": 3038
+    },
+    {
+      "epoch": 0.02250391838754915,
+      "grad_norm": 2.0227725505828857,
+      "learning_rate": 4.965133917685858e-05,
+      "loss": 0.6287,
+      "step": 3069
+    },
+    {
+      "epoch": 0.022731230694494092,
+      "grad_norm": 1.348293423652649,
+      "learning_rate": 4.9637223062514714e-05,
+      "loss": 0.63,
+      "step": 3100
+    },
+    {
+      "epoch": 0.022958543001439034,
+      "grad_norm": 1.158382773399353,
+      "learning_rate": 4.962282892045718e-05,
+      "loss": 0.6409,
+      "step": 3131
+    },
+    {
+      "epoch": 0.023185855308383973,
+      "grad_norm": 1.4399057626724243,
+      "learning_rate": 4.9608156913121904e-05,
+      "loss": 0.6419,
+      "step": 3162
+    },
+    {
+      "epoch": 0.023413167615328915,
+      "grad_norm": 1.3546380996704102,
+      "learning_rate": 4.959320720608049e-05,
+      "loss": 0.6329,
+      "step": 3193
+    },
+    {
+      "epoch": 0.023640479922273857,
+      "grad_norm": 1.4881808757781982,
+      "learning_rate": 4.9577979968038354e-05,
+      "loss": 0.6369,
+      "step": 3224
+    },
+    {
+      "epoch": 0.0238677922292188,
+      "grad_norm": 1.3750452995300293,
+      "learning_rate": 4.956247537083282e-05,
+      "loss": 0.6309,
+      "step": 3255
+    },
+    {
+      "epoch": 0.024095104536163738,
+      "grad_norm": 1.26253342628479,
+      "learning_rate": 4.9546693589431145e-05,
+      "loss": 0.6231,
+      "step": 3286
+    },
+    {
+      "epoch": 0.02432241684310868,
+      "grad_norm": 1.4811683893203735,
+      "learning_rate": 4.9530634801928595e-05,
+      "loss": 0.6271,
+      "step": 3317
+    },
+    {
+      "epoch": 0.02454972915005362,
+      "grad_norm": 1.1528241634368896,
+      "learning_rate": 4.9514299189546395e-05,
+      "loss": 0.6263,
+      "step": 3348
+    },
+    {
+      "epoch": 0.02477704145699856,
+      "grad_norm": 1.1949925422668457,
+      "learning_rate": 4.949768693662973e-05,
+      "loss": 0.6183,
+      "step": 3379
+    },
+    {
+      "epoch": 0.025004353763943502,
+      "grad_norm": 1.502016305923462,
+      "learning_rate": 4.948079823064559e-05,
+      "loss": 0.6328,
+      "step": 3410
+    },
+    {
+      "epoch": 0.025231666070888444,
+      "grad_norm": 1.2588777542114258,
+      "learning_rate": 4.946363326218074e-05,
+      "loss": 0.6352,
+      "step": 3441
+    },
+    {
+      "epoch": 0.025458978377833383,
+      "grad_norm": 1.3480185270309448,
+      "learning_rate": 4.9446192224939525e-05,
+      "loss": 0.6317,
+      "step": 3472
+    },
+    {
+      "epoch": 0.025686290684778325,
+      "grad_norm": 1.2908169031143188,
+      "learning_rate": 4.942847531574167e-05,
+      "loss": 0.6242,
+      "step": 3503
+    },
+    {
+      "epoch": 0.025913602991723267,
+      "grad_norm": 1.3300666809082031,
+      "learning_rate": 4.941048273452008e-05,
+      "loss": 0.6305,
+      "step": 3534
+    },
+    {
+      "epoch": 0.026140915298668205,
+      "grad_norm": 1.2839566469192505,
+      "learning_rate": 4.9392214684318605e-05,
+      "loss": 0.6258,
+      "step": 3565
+    },
+    {
+      "epoch": 0.026368227605613147,
+      "grad_norm": 1.3442097902297974,
+      "learning_rate": 4.93736713712897e-05,
+      "loss": 0.625,
+      "step": 3596
+    },
+    {
+      "epoch": 0.02659553991255809,
+      "grad_norm": 1.3796859979629517,
+      "learning_rate": 4.9354853004692124e-05,
+      "loss": 0.6303,
+      "step": 3627
+    },
+    {
+      "epoch": 0.026822852219503028,
+      "grad_norm": 1.2877150774002075,
+      "learning_rate": 4.93357597968886e-05,
+      "loss": 0.6225,
+      "step": 3658
+    },
+    {
+      "epoch": 0.02705016452644797,
+      "grad_norm": 1.2817054986953735,
+      "learning_rate": 4.931639196334338e-05,
+      "loss": 0.6258,
+      "step": 3689
+    },
+    {
+      "epoch": 0.027277476833392912,
+      "grad_norm": 1.364005446434021,
+      "learning_rate": 4.9296749722619826e-05,
+      "loss": 0.6203,
+      "step": 3720
+    },
+    {
+      "epoch": 0.02750478914033785,
+      "grad_norm": 1.3735343217849731,
+      "learning_rate": 4.9276833296377966e-05,
+      "loss": 0.6229,
+      "step": 3751
+    },
+    {
+      "epoch": 0.027732101447282793,
+      "grad_norm": 1.5178866386413574,
+      "learning_rate": 4.925664290937196e-05,
+      "loss": 0.6239,
+      "step": 3782
+    },
+    {
+      "epoch": 0.027959413754227735,
+      "grad_norm": 1.46648108959198,
+      "learning_rate": 4.9236178789447576e-05,
+      "loss": 0.6481,
+      "step": 3813
+    },
+    {
+      "epoch": 0.028186726061172673,
+      "grad_norm": 1.3685309886932373,
+      "learning_rate": 4.921544116753962e-05,
+      "loss": 0.6244,
+      "step": 3844
+    },
+    {
+      "epoch": 0.028414038368117615,
+      "grad_norm": 1.4738421440124512,
+      "learning_rate": 4.919443027766935e-05,
+      "loss": 0.6277,
+      "step": 3875
+    },
+    {
+      "epoch": 0.028641350675062557,
+      "grad_norm": 1.268455982208252,
+      "learning_rate": 4.91731463569418e-05,
+      "loss": 0.6314,
+      "step": 3906
+    },
+    {
+      "epoch": 0.028868662982007496,
+      "grad_norm": 1.4777989387512207,
+      "learning_rate": 4.915158964554312e-05,
+      "loss": 0.6242,
+      "step": 3937
+    },
+    {
+      "epoch": 0.029095975288952438,
+      "grad_norm": 1.5087381601333618,
+      "learning_rate": 4.912976038673786e-05,
+      "loss": 0.6063,
+      "step": 3968
+    },
+    {
+      "epoch": 0.02932328759589738,
+      "grad_norm": 1.1369556188583374,
+      "learning_rate": 4.9107658826866254e-05,
+      "loss": 0.6289,
+      "step": 3999
+    },
+    {
+      "epoch": 0.029550599902842322,
+      "grad_norm": 1.5158019065856934,
+      "learning_rate": 4.908528521534139e-05,
+      "loss": 0.6276,
+      "step": 4030
+    },
+    {
+      "epoch": 0.02977791220978726,
+      "grad_norm": 1.3511649370193481,
+      "learning_rate": 4.906263980464644e-05,
+      "loss": 0.6381,
+      "step": 4061
+    },
+    {
+      "epoch": 0.030005224516732203,
+      "grad_norm": 1.1635830402374268,
+      "learning_rate": 4.903972285033178e-05,
+      "loss": 0.6122,
+      "step": 4092
+    },
+    {
+      "epoch": 0.030232536823677145,
+      "grad_norm": 1.5354084968566895,
+      "learning_rate": 4.901653461101213e-05,
+      "loss": 0.6162,
+      "step": 4123
+    },
+    {
+      "epoch": 0.030459849130622083,
+      "grad_norm": 1.3291276693344116,
+      "learning_rate": 4.8993075348363626e-05,
+      "loss": 0.6326,
+      "step": 4154
+    },
+    {
+      "epoch": 0.030687161437567025,
+      "grad_norm": 1.4173293113708496,
+      "learning_rate": 4.896934532712084e-05,
+      "loss": 0.6222,
+      "step": 4185
+    },
+    {
+      "epoch": 0.030914473744511967,
+      "grad_norm": 1.362136721611023,
+      "learning_rate": 4.8945344815073846e-05,
+      "loss": 0.6244,
+      "step": 4216
+    },
+    {
+      "epoch": 0.031141786051456906,
+      "grad_norm": 1.4081398248672485,
+      "learning_rate": 4.892107408306516e-05,
+      "loss": 0.6145,
+      "step": 4247
+    },
+    {
+      "epoch": 0.03136909835840185,
+      "grad_norm": 1.2643316984176636,
+      "learning_rate": 4.889653340498669e-05,
+      "loss": 0.6223,
+      "step": 4278
+    },
+    {
+      "epoch": 0.03159641066534679,
+      "grad_norm": 1.2005313634872437,
+      "learning_rate": 4.8871723057776664e-05,
+      "loss": 0.6164,
+      "step": 4309
+    },
+    {
+      "epoch": 0.03182372297229173,
+      "grad_norm": 1.1127780675888062,
+      "learning_rate": 4.8846643321416476e-05,
+      "loss": 0.6149,
+      "step": 4340
+    },
+    {
+      "epoch": 0.032051035279236674,
+      "grad_norm": 1.1740052700042725,
+      "learning_rate": 4.882129447892753e-05,
+      "loss": 0.6256,
+      "step": 4371
+    },
+    {
+      "epoch": 0.03227834758618161,
+      "grad_norm": 1.3976714611053467,
+      "learning_rate": 4.8795676816368076e-05,
+      "loss": 0.6236,
+      "step": 4402
+    },
+    {
+      "epoch": 0.03250565989312655,
+      "grad_norm": 1.2942179441452026,
+      "learning_rate": 4.876979062282995e-05,
+      "loss": 0.6192,
+      "step": 4433
+    },
+    {
+      "epoch": 0.03273297220007149,
+      "grad_norm": 1.167564868927002,
+      "learning_rate": 4.8743636190435325e-05,
+      "loss": 0.6156,
+      "step": 4464
+    },
+    {
+      "epoch": 0.032960284507016435,
+      "grad_norm": 1.2276692390441895,
+      "learning_rate": 4.871721381433344e-05,
+      "loss": 0.6115,
+      "step": 4495
+    },
+    {
+      "epoch": 0.03318759681396138,
+      "grad_norm": 1.3207038640975952,
+      "learning_rate": 4.869052379269719e-05,
+      "loss": 0.6076,
+      "step": 4526
+    },
+    {
+      "epoch": 0.03341490912090632,
+      "grad_norm": 1.3339529037475586,
+      "learning_rate": 4.866356642671985e-05,
+      "loss": 0.624,
+      "step": 4557
+    },
+    {
+      "epoch": 0.033642221427851254,
+      "grad_norm": 1.2095152139663696,
+      "learning_rate": 4.8636342020611634e-05,
+      "loss": 0.6235,
+      "step": 4588
+    },
+    {
+      "epoch": 0.033869533734796196,
+      "grad_norm": 1.643731951713562,
+      "learning_rate": 4.860885088159626e-05,
+      "loss": 0.615,
+      "step": 4619
+    },
+    {
+      "epoch": 0.03409684604174114,
+      "grad_norm": 1.1702327728271484,
+      "learning_rate": 4.858109331990751e-05,
+      "loss": 0.6218,
+      "step": 4650
+    },
+    {
+      "epoch": 0.03432415834868608,
+      "grad_norm": 1.2640793323516846,
+      "learning_rate": 4.855306964878567e-05,
+      "loss": 0.6063,
+      "step": 4681
+    },
+    {
+      "epoch": 0.03455147065563102,
+      "grad_norm": 1.193090796470642,
+      "learning_rate": 4.8524780184474084e-05,
+      "loss": 0.6139,
+      "step": 4712
+    },
+    {
+      "epoch": 0.034778782962575965,
+      "grad_norm": 1.3107709884643555,
+      "learning_rate": 4.8496225246215496e-05,
+      "loss": 0.6156,
+      "step": 4743
+    },
+    {
+      "epoch": 0.0350060952695209,
+      "grad_norm": 1.9848322868347168,
+      "learning_rate": 4.8467405156248505e-05,
+      "loss": 0.6109,
+      "step": 4774
+    },
+    {
+      "epoch": 0.03523340757646584,
+      "grad_norm": 1.2142692804336548,
+      "learning_rate": 4.843832023980392e-05,
+      "loss": 0.6104,
+      "step": 4805
+    },
+    {
+      "epoch": 0.035460719883410784,
+      "grad_norm": 1.379026174545288,
+      "learning_rate": 4.840897082510106e-05,
+      "loss": 0.6111,
+      "step": 4836
+    },
+    {
+      "epoch": 0.035688032190355726,
+      "grad_norm": 1.4533346891403198,
+      "learning_rate": 4.8379357243344084e-05,
+      "loss": 0.6107,
+      "step": 4867
+    },
+    {
+      "epoch": 0.03591534449730067,
+      "grad_norm": 1.2567121982574463,
+      "learning_rate": 4.8349479828718236e-05,
+      "loss": 0.6123,
+      "step": 4898
+    },
+    {
+      "epoch": 0.03614265680424561,
+      "grad_norm": 1.764093279838562,
+      "learning_rate": 4.8319338918386075e-05,
+      "loss": 0.6248,
+      "step": 4929
+    },
+    {
+      "epoch": 0.036369969111190545,
+      "grad_norm": 1.078190565109253,
+      "learning_rate": 4.828893485248369e-05,
+      "loss": 0.6149,
+      "step": 4960
+    },
+    {
+      "epoch": 0.03659728141813549,
+      "grad_norm": 1.2174266576766968,
+      "learning_rate": 4.825826797411682e-05,
+      "loss": 0.6115,
+      "step": 4991
+    },
+    {
+      "epoch": 0.03682459372508043,
+      "grad_norm": 1.2275094985961914,
+      "learning_rate": 4.822733862935702e-05,
+      "loss": 0.6051,
+      "step": 5022
+    },
+    {
+      "epoch": 0.03705190603202537,
+      "grad_norm": 1.526706337928772,
+      "learning_rate": 4.819614716723775e-05,
+      "loss": 0.6073,
+      "step": 5053
+    },
+    {
+      "epoch": 0.03727921833897031,
+      "grad_norm": 1.1073312759399414,
+      "learning_rate": 4.8164693939750425e-05,
+      "loss": 0.6071,
+      "step": 5084
+    },
+    {
+      "epoch": 0.037506530645915255,
+      "grad_norm": 1.494463562965393,
+      "learning_rate": 4.813297930184042e-05,
+      "loss": 0.6014,
+      "step": 5115
+    },
+    {
+      "epoch": 0.0377338429528602,
+      "grad_norm": 1.1264615058898926,
+      "learning_rate": 4.810100361140314e-05,
+      "loss": 0.6201,
+      "step": 5146
+    },
+    {
+      "epoch": 0.03796115525980513,
+      "grad_norm": 1.2129037380218506,
+      "learning_rate": 4.8068767229279885e-05,
+      "loss": 0.5998,
+      "step": 5177
+    },
+    {
+      "epoch": 0.038188467566750074,
+      "grad_norm": 1.2444956302642822,
+      "learning_rate": 4.8036270519253854e-05,
+      "loss": 0.6144,
+      "step": 5208
+    },
+    {
+      "epoch": 0.038415779873695016,
+      "grad_norm": 1.311604380607605,
+      "learning_rate": 4.8003513848046e-05,
+      "loss": 0.606,
+      "step": 5239
+    },
+    {
+      "epoch": 0.03864309218063996,
+      "grad_norm": 1.2056453227996826,
+      "learning_rate": 4.79704975853109e-05,
+      "loss": 0.6088,
+      "step": 5270
+    },
+    {
+      "epoch": 0.0388704044875849,
+      "grad_norm": 1.4589018821716309,
+      "learning_rate": 4.793722210363262e-05,
+      "loss": 0.6116,
+      "step": 5301
+    },
+    {
+      "epoch": 0.03909771679452984,
+      "grad_norm": 1.1214395761489868,
+      "learning_rate": 4.7903687778520414e-05,
+      "loss": 0.6215,
+      "step": 5332
+    },
+    {
+      "epoch": 0.03932502910147478,
+      "grad_norm": 1.3108386993408203,
+      "learning_rate": 4.7869894988404593e-05,
+      "loss": 0.6182,
+      "step": 5363
+    },
+    {
+      "epoch": 0.03955234140841972,
+      "grad_norm": 1.3839397430419922,
+      "learning_rate": 4.783584411463221e-05,
+      "loss": 0.5985,
+      "step": 5394
+    },
+    {
+      "epoch": 0.03977965371536466,
+      "grad_norm": 1.2053918838500977,
+      "learning_rate": 4.780153554146274e-05,
+      "loss": 0.6036,
+      "step": 5425
+    },
+    {
+      "epoch": 0.040006966022309604,
+      "grad_norm": 1.1878604888916016,
+      "learning_rate": 4.7766969656063766e-05,
+      "loss": 0.6086,
+      "step": 5456
+    },
+    {
+      "epoch": 0.040234278329254546,
+      "grad_norm": 1.2663103342056274,
+      "learning_rate": 4.773214684850662e-05,
+      "loss": 0.6105,
+      "step": 5487
+    },
+    {
+      "epoch": 0.04046159063619949,
+      "grad_norm": 1.513929009437561,
+      "learning_rate": 4.769706751176193e-05,
+      "loss": 0.619,
+      "step": 5518
+    },
+    {
+      "epoch": 0.04068890294314442,
+      "grad_norm": 1.1540343761444092,
+      "learning_rate": 4.7661732041695264e-05,
+      "loss": 0.6054,
+      "step": 5549
+    },
+    {
+      "epoch": 0.040916215250089365,
+      "grad_norm": 1.1649845838546753,
+      "learning_rate": 4.762614083706258e-05,
+      "loss": 0.614,
+      "step": 5580
+    },
+    {
+      "epoch": 0.04114352755703431,
+      "grad_norm": 1.2354192733764648,
+      "learning_rate": 4.759029429950581e-05,
+      "loss": 0.5996,
+      "step": 5611
+    },
+    {
+      "epoch": 0.04137083986397925,
+      "grad_norm": 1.3275151252746582,
+      "learning_rate": 4.7554192833548235e-05,
+      "loss": 0.6001,
+      "step": 5642
+    },
+    {
+      "epoch": 0.04159815217092419,
+      "grad_norm": 1.2199240922927856,
+      "learning_rate": 4.751783684659e-05,
+      "loss": 0.5944,
+      "step": 5673
+    },
+    {
+      "epoch": 0.04182546447786913,
+      "grad_norm": 1.213654637336731,
+      "learning_rate": 4.748122674890348e-05,
+      "loss": 0.6129,
+      "step": 5704
+    },
+    {
+      "epoch": 0.04205277678481407,
+      "grad_norm": 1.1168550252914429,
+      "learning_rate": 4.7444362953628654e-05,
+      "loss": 0.608,
+      "step": 5735
+    },
+    {
+      "epoch": 0.04228008909175901,
+      "grad_norm": 1.0786912441253662,
+      "learning_rate": 4.7407245876768424e-05,
+      "loss": 0.6092,
+      "step": 5766
+    },
+    {
+      "epoch": 0.04250740139870395,
+      "grad_norm": 1.2462965250015259,
+      "learning_rate": 4.736987593718397e-05,
+      "loss": 0.5873,
+      "step": 5797
+    },
+    {
+      "epoch": 0.042734713705648894,
+      "grad_norm": 1.2630695104599,
+      "learning_rate": 4.733225355658999e-05,
+      "loss": 0.5943,
+      "step": 5828
+    },
+    {
+      "epoch": 0.042962026012593836,
+      "grad_norm": 1.3114591836929321,
+      "learning_rate": 4.7294379159549926e-05,
+      "loss": 0.6,
+      "step": 5859
+    },
+    {
+      "epoch": 0.04318933831953878,
+      "grad_norm": 1.3034451007843018,
+      "learning_rate": 4.725625317347119e-05,
+      "loss": 0.6076,
+      "step": 5890
+    },
+    {
+      "epoch": 0.04341665062648372,
+      "grad_norm": 1.234309434890747,
+      "learning_rate": 4.7217876028600374e-05,
+      "loss": 0.6054,
+      "step": 5921
+    },
+    {
+      "epoch": 0.043643962933428655,
+      "grad_norm": 1.286466121673584,
+      "learning_rate": 4.717924815801832e-05,
+      "loss": 0.5974,
+      "step": 5952
+    },
+    {
+      "epoch": 0.0438712752403736,
+      "grad_norm": 1.1768263578414917,
+      "learning_rate": 4.714036999763532e-05,
+      "loss": 0.5945,
+      "step": 5983
+    },
+    {
+      "epoch": 0.04409858754731854,
+      "grad_norm": 1.0463429689407349,
+      "learning_rate": 4.7101241986186116e-05,
+      "loss": 0.6049,
+      "step": 6014
+    },
+    {
+      "epoch": 0.04432589985426348,
+      "grad_norm": 1.0957670211791992,
+      "learning_rate": 4.7061864565225e-05,
+      "loss": 0.6014,
+      "step": 6045
+    },
+    {
+      "epoch": 0.04455321216120842,
+      "grad_norm": 1.2231004238128662,
+      "learning_rate": 4.702223817912081e-05,
+      "loss": 0.6085,
+      "step": 6076
+    },
+    {
+      "epoch": 0.044780524468153365,
+      "grad_norm": 1.2427260875701904,
+      "learning_rate": 4.698236327505195e-05,
+      "loss": 0.5926,
+      "step": 6107
+    },
+    {
+      "epoch": 0.0450078367750983,
+      "grad_norm": 1.1379714012145996,
+      "learning_rate": 4.694224030300127e-05,
+      "loss": 0.6094,
+      "step": 6138
+    },
+    {
+      "epoch": 0.04523514908204324,
+      "grad_norm": 1.2138813734054565,
+      "learning_rate": 4.690186971575107e-05,
+      "loss": 0.5927,
+      "step": 6169
+    },
+    {
+      "epoch": 0.045462461388988185,
+      "grad_norm": 1.1706838607788086,
+      "learning_rate": 4.6861251968877916e-05,
+      "loss": 0.5938,
+      "step": 6200
+    },
+    {
+      "epoch": 0.04568977369593313,
+      "grad_norm": 1.2214962244033813,
+      "learning_rate": 4.68203875207476e-05,
+      "loss": 0.5983,
+      "step": 6231
+    },
+    {
+      "epoch": 0.04591708600287807,
+      "grad_norm": 1.2631845474243164,
+      "learning_rate": 4.677927683250983e-05,
+      "loss": 0.6058,
+      "step": 6262
+    },
+    {
+      "epoch": 0.04614439830982301,
+      "grad_norm": 1.1130115985870361,
+      "learning_rate": 4.6737920368093156e-05,
+      "loss": 0.5974,
+      "step": 6293
+    },
+    {
+      "epoch": 0.046371710616767946,
+      "grad_norm": 1.2656989097595215,
+      "learning_rate": 4.669631859419965e-05,
+      "loss": 0.5854,
+      "step": 6324
+    },
+    {
+      "epoch": 0.04659902292371289,
+      "grad_norm": 1.0462357997894287,
+      "learning_rate": 4.6654471980299676e-05,
+      "loss": 0.5972,
+      "step": 6355
+    },
+    {
+      "epoch": 0.04682633523065783,
+      "grad_norm": 1.145822525024414,
+      "learning_rate": 4.661238099862658e-05,
+      "loss": 0.582,
+      "step": 6386
+    },
+    {
+      "epoch": 0.04705364753760277,
+      "grad_norm": 1.163480520248413,
+      "learning_rate": 4.657004612417138e-05,
+      "loss": 0.589,
+      "step": 6417
+    },
+    {
+      "epoch": 0.047280959844547714,
+      "grad_norm": 1.2010438442230225,
+      "learning_rate": 4.6527467834677374e-05,
+      "loss": 0.5863,
+      "step": 6448
+    },
+    {
+      "epoch": 0.047508272151492656,
+      "grad_norm": 1.119146704673767,
+      "learning_rate": 4.648464661063478e-05,
+      "loss": 0.5909,
+      "step": 6479
+    },
+    {
+      "epoch": 0.0477355844584376,
+      "grad_norm": 1.1676424741744995,
+      "learning_rate": 4.6441582935275264e-05,
+      "loss": 0.5791,
+      "step": 6510
+    },
+    {
+      "epoch": 0.04796289676538253,
+      "grad_norm": 1.0956594944000244,
+      "learning_rate": 4.6398277294566586e-05,
+      "loss": 0.6045,
+      "step": 6541
+    },
+    {
+      "epoch": 0.048190209072327475,
+      "grad_norm": 1.1999818086624146,
+      "learning_rate": 4.6354730177207e-05,
+      "loss": 0.6141,
+      "step": 6572
+    },
+    {
+      "epoch": 0.04841752137927242,
+      "grad_norm": 1.130088210105896,
+      "learning_rate": 4.6310942074619787e-05,
+      "loss": 0.6013,
+      "step": 6603
+    },
+    {
+      "epoch": 0.04864483368621736,
+      "grad_norm": 1.1956254243850708,
+      "learning_rate": 4.626691348094777e-05,
+      "loss": 0.5873,
+      "step": 6634
+    },
+    {
+      "epoch": 0.0488721459931623,
+      "grad_norm": 1.1242092847824097,
+      "learning_rate": 4.622264489304762e-05,
+      "loss": 0.5898,
+      "step": 6665
+    },
+    {
+      "epoch": 0.04909945830010724,
+      "grad_norm": 1.2921823263168335,
+      "learning_rate": 4.617813681048434e-05,
+      "loss": 0.5915,
+      "step": 6696
+    },
+    {
+      "epoch": 0.04932677060705218,
+      "grad_norm": 1.1597084999084473,
+      "learning_rate": 4.61333897355256e-05,
+      "loss": 0.5872,
+      "step": 6727
+    },
+    {
+      "epoch": 0.04955408291399712,
+      "grad_norm": 1.1626496315002441,
+      "learning_rate": 4.608840417313604e-05,
+      "loss": 0.5958,
+      "step": 6758
+    },
+    {
+      "epoch": 0.04978139522094206,
+      "grad_norm": 1.1626874208450317,
+      "learning_rate": 4.6043180630971646e-05,
+      "loss": 0.5845,
+      "step": 6789
+    },
+    {
+      "epoch": 0.050008707527887004,
+      "grad_norm": 1.1668806076049805,
+      "learning_rate": 4.599771961937391e-05,
+      "loss": 0.5916,
+      "step": 6820
+    },
+    {
+      "epoch": 0.050236019834831946,
+      "grad_norm": 1.0241467952728271,
+      "learning_rate": 4.5952021651364204e-05,
+      "loss": 0.5995,
+      "step": 6851
+    },
+    {
+      "epoch": 0.05046333214177689,
+      "grad_norm": 1.220924973487854,
+      "learning_rate": 4.590608724263786e-05,
+      "loss": 0.6052,
+      "step": 6882
+    },
+    {
+      "epoch": 0.050690644448721824,
+      "grad_norm": 1.199133038520813,
+      "learning_rate": 4.585991691155845e-05,
+      "loss": 0.5983,
+      "step": 6913
+    },
+    {
+      "epoch": 0.050917956755666766,
+      "grad_norm": 1.1263436079025269,
+      "learning_rate": 4.581351117915188e-05,
+      "loss": 0.5944,
+      "step": 6944
+    },
+    {
+      "epoch": 0.05114526906261171,
+      "grad_norm": 1.1131410598754883,
+      "learning_rate": 4.5766870569100534e-05,
+      "loss": 0.6013,
+      "step": 6975
+    },
+    {
+      "epoch": 0.05137258136955665,
+      "grad_norm": 1.2235490083694458,
+      "learning_rate": 4.571999560773736e-05,
+      "loss": 0.5906,
+      "step": 7006
+    },
+    {
+      "epoch": 0.05159989367650159,
+      "grad_norm": 1.0688278675079346,
+      "learning_rate": 4.5672886824039915e-05,
+      "loss": 0.5976,
+      "step": 7037
+    },
+    {
+      "epoch": 0.051827205983446534,
+      "grad_norm": 1.084426999092102,
+      "learning_rate": 4.5625544749624435e-05,
+      "loss": 0.5973,
+      "step": 7068
+    },
+    {
+      "epoch": 0.05205451829039147,
+      "grad_norm": 1.1357454061508179,
+      "learning_rate": 4.5577969918739794e-05,
+      "loss": 0.5925,
+      "step": 7099
+    },
+    {
+      "epoch": 0.05228183059733641,
+      "grad_norm": 1.0364577770233154,
+      "learning_rate": 4.5530162868261486e-05,
+      "loss": 0.598,
+      "step": 7130
+    },
+    {
+      "epoch": 0.05250914290428135,
+      "grad_norm": 1.223486304283142,
+      "learning_rate": 4.548212413768558e-05,
+      "loss": 0.6044,
+      "step": 7161
+    },
+    {
+      "epoch": 0.052736455211226295,
+      "grad_norm": 1.286464810371399,
+      "learning_rate": 4.543385426912261e-05,
+      "loss": 0.6038,
+      "step": 7192
+    },
+    {
+      "epoch": 0.05296376751817124,
+      "grad_norm": 1.169421672821045,
+      "learning_rate": 4.53853538072915e-05,
+      "loss": 0.5989,
+      "step": 7223
+    },
+    {
+      "epoch": 0.05319107982511618,
+      "grad_norm": 1.0416791439056396,
+      "learning_rate": 4.533662329951336e-05,
+      "loss": 0.5991,
+      "step": 7254
+    },
+    {
+      "epoch": 0.05341839213206112,
+      "grad_norm": 1.1406258344650269,
+      "learning_rate": 4.528766329570536e-05,
+      "loss": 0.5924,
+      "step": 7285
+    },
+    {
+      "epoch": 0.053645704439006056,
+      "grad_norm": 1.0331827402114868,
+      "learning_rate": 4.523847434837447e-05,
+      "loss": 0.5883,
+      "step": 7316
+    },
+    {
+      "epoch": 0.053873016745951,
+      "grad_norm": 1.1116039752960205,
+      "learning_rate": 4.518905701261128e-05,
+      "loss": 0.5866,
+      "step": 7347
+    },
+    {
+      "epoch": 0.05410032905289594,
+      "grad_norm": 1.0617607831954956,
+      "learning_rate": 4.5139411846083715e-05,
+      "loss": 0.585,
+      "step": 7378
+    },
+    {
+      "epoch": 0.05432764135984088,
+      "grad_norm": 1.1014251708984375,
+      "learning_rate": 4.508953940903073e-05,
+      "loss": 0.5817,
+      "step": 7409
+    },
+    {
+      "epoch": 0.054554953666785824,
+      "grad_norm": 1.083156943321228,
+      "learning_rate": 4.5039440264255994e-05,
+      "loss": 0.5819,
+      "step": 7440
+    },
+    {
+      "epoch": 0.054782265973730766,
+      "grad_norm": 1.1377294063568115,
+      "learning_rate": 4.498911497712155e-05,
+      "loss": 0.6036,
+      "step": 7471
+    },
+    {
+      "epoch": 0.0550095782806757,
+      "grad_norm": 1.173553228378296,
+      "learning_rate": 4.493856411554142e-05,
+      "loss": 0.5844,
+      "step": 7502
+    },
+    {
+      "epoch": 0.05523689058762064,
+      "grad_norm": 1.076294183731079,
+      "learning_rate": 4.4887788249975206e-05,
+      "loss": 0.5838,
+      "step": 7533
+    },
+    {
+      "epoch": 0.055464202894565585,
+      "grad_norm": 1.100803256034851,
+      "learning_rate": 4.4836787953421656e-05,
+      "loss": 0.5869,
+      "step": 7564
+    },
+    {
+      "epoch": 0.05569151520151053,
+      "grad_norm": 1.1871910095214844,
+      "learning_rate": 4.478556380141218e-05,
+      "loss": 0.5951,
+      "step": 7595
+    },
+    {
+      "epoch": 0.05591882750845547,
+      "grad_norm": 1.27151358127594,
+      "learning_rate": 4.4734116372004375e-05,
+      "loss": 0.5865,
+      "step": 7626
+    },
+    {
+      "epoch": 0.05614613981540041,
+      "grad_norm": 1.2775440216064453,
+      "learning_rate": 4.4682446245775477e-05,
+      "loss": 0.586,
+      "step": 7657
+    },
+    {
+      "epoch": 0.05637345212234535,
+      "grad_norm": 1.2034847736358643,
+      "learning_rate": 4.463055400581586e-05,
+      "loss": 0.6005,
+      "step": 7688
+    },
+    {
+      "epoch": 0.05660076442929029,
+      "grad_norm": 1.8479490280151367,
+      "learning_rate": 4.4578440237722374e-05,
+      "loss": 0.5797,
+      "step": 7719
+    },
+    {
+      "epoch": 0.05682807673623523,
+      "grad_norm": 1.1473543643951416,
+      "learning_rate": 4.452610552959183e-05,
+      "loss": 0.5818,
+      "step": 7750
+    },
+    {
+      "epoch": 0.05705538904318017,
+      "grad_norm": 1.1030511856079102,
+      "learning_rate": 4.447355047201428e-05,
+      "loss": 0.5886,
+      "step": 7781
+    },
+    {
+      "epoch": 0.057282701350125115,
+      "grad_norm": 1.0135844945907593,
+      "learning_rate": 4.4420775658066414e-05,
+      "loss": 0.5921,
+      "step": 7812
+    },
+    {
+      "epoch": 0.05751001365707006,
+      "grad_norm": 1.0830888748168945,
+      "learning_rate": 4.436778168330484e-05,
+      "loss": 0.5835,
+      "step": 7843
+    },
+    {
+      "epoch": 0.05773732596401499,
+      "grad_norm": 1.1242387294769287,
+      "learning_rate": 4.4314569145759353e-05,
+      "loss": 0.5787,
+      "step": 7874
+    },
+    {
+      "epoch": 0.057964638270959934,
+      "grad_norm": 1.0822036266326904,
+      "learning_rate": 4.42611386459262e-05,
+      "loss": 0.5773,
+      "step": 7905
+    },
+    {
+      "epoch": 0.058191950577904876,
+      "grad_norm": 1.1694915294647217,
+      "learning_rate": 4.420749078676133e-05,
+      "loss": 0.5924,
+      "step": 7936
+    },
+    {
+      "epoch": 0.05841926288484982,
+      "grad_norm": 1.0987952947616577,
+      "learning_rate": 4.4153626173673516e-05,
+      "loss": 0.5863,
+      "step": 7967
+    },
+    {
+      "epoch": 0.05864657519179476,
+      "grad_norm": 1.1604340076446533,
+      "learning_rate": 4.409954541451762e-05,
+      "loss": 0.5754,
+      "step": 7998
+    },
+    {
+      "epoch": 0.0588738874987397,
+      "grad_norm": 1.1734123229980469,
+      "learning_rate": 4.404524911958764e-05,
+      "loss": 0.5868,
+      "step": 8029
+    },
+    {
+      "epoch": 0.059101199805684644,
+      "grad_norm": 1.2483155727386475,
+      "learning_rate": 4.399073790160989e-05,
+      "loss": 0.5891,
+      "step": 8060
+    },
+    {
+      "epoch": 0.05932851211262958,
+      "grad_norm": 1.1200464963912964,
+      "learning_rate": 4.393601237573607e-05,
+      "loss": 0.588,
+      "step": 8091
+    },
+    {
+      "epoch": 0.05955582441957452,
+      "grad_norm": 1.6909037828445435,
+      "learning_rate": 4.388107315953628e-05,
+      "loss": 0.5797,
+      "step": 8122
+    },
+    {
+      "epoch": 0.05978313672651946,
+      "grad_norm": 1.0194246768951416,
+      "learning_rate": 4.382592087299212e-05,
+      "loss": 0.5823,
+      "step": 8153
+    },
+    {
+      "epoch": 0.060010449033464405,
+      "grad_norm": 1.341172456741333,
+      "learning_rate": 4.377055613848964e-05,
+      "loss": 0.5782,
+      "step": 8184
+    },
+    {
+      "epoch": 0.06023776134040935,
+      "grad_norm": 1.0860446691513062,
+      "learning_rate": 4.3714979580812355e-05,
+      "loss": 0.603,
+      "step": 8215
+    },
+    {
+      "epoch": 0.06046507364735429,
+      "grad_norm": 0.9628092646598816,
+      "learning_rate": 4.365919182713416e-05,
+      "loss": 0.5809,
+      "step": 8246
+    },
+    {
+      "epoch": 0.060692385954299224,
+      "grad_norm": 1.0186283588409424,
+      "learning_rate": 4.360319350701226e-05,
+      "loss": 0.5812,
+      "step": 8277
+    },
+    {
+      "epoch": 0.060919698261244166,
+      "grad_norm": 1.1478028297424316,
+      "learning_rate": 4.3546985252380115e-05,
+      "loss": 0.5887,
+      "step": 8308
+    },
+    {
+      "epoch": 0.06114701056818911,
+      "grad_norm": 1.0377260446548462,
+      "learning_rate": 4.349056769754021e-05,
+      "loss": 0.5824,
+      "step": 8339
+    },
+    {
+      "epoch": 0.06137432287513405,
+      "grad_norm": 1.1011360883712769,
+      "learning_rate": 4.3433941479156994e-05,
+      "loss": 0.591,
+      "step": 8370
+    },
+    {
+      "epoch": 0.06160163518207899,
+      "grad_norm": 1.2130680084228516,
+      "learning_rate": 4.3377107236249647e-05,
+      "loss": 0.583,
+      "step": 8401
+    },
+    {
+      "epoch": 0.061828947489023935,
+      "grad_norm": 1.0764377117156982,
+      "learning_rate": 4.332006561018488e-05,
+      "loss": 0.5883,
+      "step": 8432
+    },
+    {
+      "epoch": 0.06205625979596887,
+      "grad_norm": 1.2170931100845337,
+      "learning_rate": 4.3262817244669683e-05,
+      "loss": 0.5877,
+      "step": 8463
+    },
+    {
+      "epoch": 0.06228357210291381,
+      "grad_norm": 1.2120672464370728,
+      "learning_rate": 4.3205362785744083e-05,
+      "loss": 0.5945,
+      "step": 8494
+    },
+    {
+      "epoch": 0.06251088440985876,
+      "grad_norm": 1.1120613813400269,
+      "learning_rate": 4.314770288177384e-05,
+      "loss": 0.5844,
+      "step": 8525
+    },
+    {
+      "epoch": 0.0627381967168037,
+      "grad_norm": 1.1696306467056274,
+      "learning_rate": 4.308983818344313e-05,
+      "loss": 0.5848,
+      "step": 8556
+    },
+    {
+      "epoch": 0.06296550902374863,
+      "grad_norm": 0.9421460032463074,
+      "learning_rate": 4.3031769343747206e-05,
+      "loss": 0.5715,
+      "step": 8587
+    },
+    {
+      "epoch": 0.06319282133069358,
+      "grad_norm": 1.022047996520996,
+      "learning_rate": 4.297349701798505e-05,
+      "loss": 0.5846,
+      "step": 8618
+    },
+    {
+      "epoch": 0.06342013363763851,
+      "grad_norm": 1.0041323900222778,
+      "learning_rate": 4.2915021863751916e-05,
+      "loss": 0.5907,
+      "step": 8649
+    },
+    {
+      "epoch": 0.06364744594458346,
+      "grad_norm": 1.0225340127944946,
+      "learning_rate": 4.285634454093198e-05,
+      "loss": 0.5775,
+      "step": 8680
+    },
+    {
+      "epoch": 0.0638747582515284,
+      "grad_norm": 1.0869609117507935,
+      "learning_rate": 4.279746571169086e-05,
+      "loss": 0.5777,
+      "step": 8711
+    },
+    {
+      "epoch": 0.06410207055847335,
+      "grad_norm": 1.1217941045761108,
+      "learning_rate": 4.2738386040468136e-05,
+      "loss": 0.5754,
+      "step": 8742
+    },
+    {
+      "epoch": 0.06432938286541828,
+      "grad_norm": 1.0278992652893066,
+      "learning_rate": 4.2679106193969866e-05,
+      "loss": 0.5901,
+      "step": 8773
+    },
+    {
+      "epoch": 0.06455669517236322,
+      "grad_norm": 1.0391658544540405,
+      "learning_rate": 4.261962684116106e-05,
+      "loss": 0.5794,
+      "step": 8804
+    },
+    {
+      "epoch": 0.06478400747930817,
+      "grad_norm": 1.3602745532989502,
+      "learning_rate": 4.2559948653258145e-05,
+      "loss": 0.5954,
+      "step": 8835
+    },
+    {
+      "epoch": 0.0650113197862531,
+      "grad_norm": 1.0198665857315063,
+      "learning_rate": 4.250007230372134e-05,
+      "loss": 0.5828,
+      "step": 8866
+    },
+    {
+      "epoch": 0.06523863209319805,
+      "grad_norm": 0.9927871823310852,
+      "learning_rate": 4.2439998468247126e-05,
+      "loss": 0.5781,
+      "step": 8897
+    },
+    {
+      "epoch": 0.06546594440014299,
+      "grad_norm": 1.0893994569778442,
+      "learning_rate": 4.2379727824760566e-05,
+      "loss": 0.5787,
+      "step": 8928
+    },
+    {
+      "epoch": 0.06569325670708792,
+      "grad_norm": 1.1195639371871948,
+      "learning_rate": 4.231926105340768e-05,
+      "loss": 0.5873,
+      "step": 8959
+    },
+    {
+      "epoch": 0.06592056901403287,
+      "grad_norm": 1.1602458953857422,
+      "learning_rate": 4.225859883654776e-05,
+      "loss": 0.5823,
+      "step": 8990
+    },
+    {
+      "epoch": 0.0661478813209778,
+      "grad_norm": 0.9547759890556335,
+      "learning_rate": 4.219774185874569e-05,
+      "loss": 0.5765,
+      "step": 9021
+    },
+    {
+      "epoch": 0.06637519362792275,
+      "grad_norm": 1.0450401306152344,
+      "learning_rate": 4.213669080676418e-05,
+      "loss": 0.575,
+      "step": 9052
+    },
+    {
+      "epoch": 0.06660250593486769,
+      "grad_norm": 1.0602561235427856,
+      "learning_rate": 4.2075446369556056e-05,
+      "loss": 0.5893,
+      "step": 9083
+    },
+    {
+      "epoch": 0.06682981824181264,
+      "grad_norm": 1.1700832843780518,
+      "learning_rate": 4.201400923825648e-05,
+      "loss": 0.5753,
+      "step": 9114
+    },
+    {
+      "epoch": 0.06705713054875757,
+      "grad_norm": 1.0414587259292603,
+      "learning_rate": 4.195238010617511e-05,
+      "loss": 0.5812,
+      "step": 9145
+    },
+    {
+      "epoch": 0.06728444285570251,
+      "grad_norm": 0.9599168300628662,
+      "learning_rate": 4.1890559668788344e-05,
+      "loss": 0.5815,
+      "step": 9176
+    },
+    {
+      "epoch": 0.06751175516264746,
+      "grad_norm": 1.0416408777236938,
+      "learning_rate": 4.1828548623731405e-05,
+      "loss": 0.5771,
+      "step": 9207
+    },
+    {
+      "epoch": 0.06773906746959239,
+      "grad_norm": 1.0418208837509155,
+      "learning_rate": 4.1766347670790506e-05,
+      "loss": 0.5767,
+      "step": 9238
+    },
+    {
+      "epoch": 0.06796637977653734,
+      "grad_norm": 0.916523814201355,
+      "learning_rate": 4.170395751189495e-05,
+      "loss": 0.5822,
+      "step": 9269
+    },
+    {
+      "epoch": 0.06819369208348228,
+      "grad_norm": 1.0706210136413574,
+      "learning_rate": 4.164137885110921e-05,
+      "loss": 0.5674,
+      "step": 9300
+    },
+    {
+      "epoch": 0.06842100439042721,
+      "grad_norm": 1.0478285551071167,
+      "learning_rate": 4.157861239462495e-05,
+      "loss": 0.568,
+      "step": 9331
+    },
+    {
+      "epoch": 0.06864831669737216,
+      "grad_norm": 1.0983623266220093,
+      "learning_rate": 4.1515658850753114e-05,
+      "loss": 0.5911,
+      "step": 9362
+    },
+    {
+      "epoch": 0.0688756290043171,
+      "grad_norm": 0.986751139163971,
+      "learning_rate": 4.145251892991588e-05,
+      "loss": 0.5803,
+      "step": 9393
+    },
+    {
+      "epoch": 0.06910294131126204,
+      "grad_norm": 1.1188838481903076,
+      "learning_rate": 4.138919334463868e-05,
+      "loss": 0.5766,
+      "step": 9424
+    },
+    {
+      "epoch": 0.06933025361820698,
+      "grad_norm": 0.9229809045791626,
+      "learning_rate": 4.1325682809542124e-05,
+      "loss": 0.5652,
+      "step": 9455
+    },
+    {
+      "epoch": 0.06955756592515193,
+      "grad_norm": 1.2419557571411133,
+      "learning_rate": 4.126198804133398e-05,
+      "loss": 0.5856,
+      "step": 9486
+    },
+    {
+      "epoch": 0.06978487823209686,
+      "grad_norm": 1.0183296203613281,
+      "learning_rate": 4.1198109758801055e-05,
+      "loss": 0.5863,
+      "step": 9517
+    },
+    {
+      "epoch": 0.0700121905390418,
+      "grad_norm": 1.1026153564453125,
+      "learning_rate": 4.113404868280107e-05,
+      "loss": 0.5772,
+      "step": 9548
+    },
+    {
+      "epoch": 0.07023950284598675,
+      "grad_norm": 1.0023332834243774,
+      "learning_rate": 4.106980553625457e-05,
+      "loss": 0.5682,
+      "step": 9579
+    },
+    {
+      "epoch": 0.07046681515293168,
+      "grad_norm": 1.0359746217727661,
+      "learning_rate": 4.100538104413674e-05,
+      "loss": 0.5698,
+      "step": 9610
+    },
+    {
+      "epoch": 0.07069412745987663,
+      "grad_norm": 1.005833387374878,
+      "learning_rate": 4.09407759334692e-05,
+      "loss": 0.5722,
+      "step": 9641
+    },
+    {
+      "epoch": 0.07092143976682157,
+      "grad_norm": 1.0719865560531616,
+      "learning_rate": 4.087599093331186e-05,
+      "loss": 0.5938,
+      "step": 9672
+    },
+    {
+      "epoch": 0.07114875207376652,
+      "grad_norm": 1.2361112833023071,
+      "learning_rate": 4.081102677475462e-05,
+      "loss": 0.5754,
+      "step": 9703
+    },
+    {
+      "epoch": 0.07137606438071145,
+      "grad_norm": 1.0325888395309448,
+      "learning_rate": 4.0745884190909194e-05,
+      "loss": 0.5823,
+      "step": 9734
+    },
+    {
+      "epoch": 0.07160337668765639,
+      "grad_norm": 1.0897630453109741,
+      "learning_rate": 4.0680563916900796e-05,
+      "loss": 0.5776,
+      "step": 9765
+    },
+    {
+      "epoch": 0.07183068899460134,
+      "grad_norm": 1.0404136180877686,
+      "learning_rate": 4.0615066689859815e-05,
+      "loss": 0.5741,
+      "step": 9796
+    },
+    {
+      "epoch": 0.07205800130154627,
+      "grad_norm": 1.187201738357544,
+      "learning_rate": 4.0549393248913584e-05,
+      "loss": 0.5761,
+      "step": 9827
+    },
+    {
+      "epoch": 0.07228531360849122,
+      "grad_norm": 0.9785891175270081,
+      "learning_rate": 4.048354433517794e-05,
+      "loss": 0.5891,
+      "step": 9858
+    },
+    {
+      "epoch": 0.07251262591543615,
+      "grad_norm": 0.9603219628334045,
+      "learning_rate": 4.0417520691748916e-05,
+      "loss": 0.5667,
+      "step": 9889
+    },
+    {
+      "epoch": 0.07273993822238109,
+      "grad_norm": 0.9943910837173462,
+      "learning_rate": 4.035132306369438e-05,
+      "loss": 0.5773,
+      "step": 9920
+    },
+    {
+      "epoch": 0.07296725052932604,
+      "grad_norm": 0.9425970911979675,
+      "learning_rate": 4.028495219804555e-05,
+      "loss": 0.5783,
+      "step": 9951
+    },
+    {
+      "epoch": 0.07319456283627097,
+      "grad_norm": 0.986157238483429,
+      "learning_rate": 4.021840884378864e-05,
+      "loss": 0.5799,
+      "step": 9982
+    },
+    {
+      "epoch": 0.07342187514321592,
+      "grad_norm": 1.0555075407028198,
+      "learning_rate": 4.015169375185633e-05,
+      "loss": 0.5946,
+      "step": 10013
+    },
+    {
+      "epoch": 0.07364918745016086,
+      "grad_norm": 0.9671381115913391,
+      "learning_rate": 4.0084807675119396e-05,
+      "loss": 0.5857,
+      "step": 10044
+    },
+    {
+      "epoch": 0.0738764997571058,
+      "grad_norm": 0.9829468131065369,
+      "learning_rate": 4.0017751368378106e-05,
+      "loss": 0.5791,
+      "step": 10075
+    },
+    {
+      "epoch": 0.07410381206405074,
+      "grad_norm": 0.8816218376159668,
+      "learning_rate": 3.995052558835377e-05,
+      "loss": 0.5772,
+      "step": 10106
+    },
+    {
+      "epoch": 0.07433112437099568,
+      "grad_norm": 1.0396629571914673,
+      "learning_rate": 3.988313109368017e-05,
+      "loss": 0.5848,
+      "step": 10137
+    },
+    {
+      "epoch": 0.07455843667794063,
+      "grad_norm": 1.0646713972091675,
+      "learning_rate": 3.981556864489504e-05,
+      "loss": 0.5702,
+      "step": 10168
+    },
+    {
+      "epoch": 0.07478574898488556,
+      "grad_norm": 1.2143402099609375,
+      "learning_rate": 3.974783900443142e-05,
+      "loss": 0.5804,
+      "step": 10199
+    },
+    {
+      "epoch": 0.07501306129183051,
+      "grad_norm": 0.9640632271766663,
+      "learning_rate": 3.9679942936609095e-05,
+      "loss": 0.5634,
+      "step": 10230
+    },
+    {
+      "epoch": 0.07524037359877545,
+      "grad_norm": 1.0565896034240723,
+      "learning_rate": 3.961188120762596e-05,
+      "loss": 0.5747,
+      "step": 10261
+    },
+    {
+      "epoch": 0.0754676859057204,
+      "grad_norm": 1.0120985507965088,
+      "learning_rate": 3.954365458554938e-05,
+      "loss": 0.5647,
+      "step": 10292
+    },
+    {
+      "epoch": 0.07569499821266533,
+      "grad_norm": 0.9489086866378784,
+      "learning_rate": 3.947526384030751e-05,
+      "loss": 0.58,
+      "step": 10323
+    },
+    {
+      "epoch": 0.07592231051961026,
+      "grad_norm": 1.0742446184158325,
+      "learning_rate": 3.9406709743680624e-05,
+      "loss": 0.5742,
+      "step": 10354
+    },
+    {
+      "epoch": 0.07614962282655521,
+      "grad_norm": 1.0096065998077393,
+      "learning_rate": 3.9337993069292366e-05,
+      "loss": 0.5702,
+      "step": 10385
+    },
+    {
+      "epoch": 0.07637693513350015,
+      "grad_norm": 1.0610495805740356,
+      "learning_rate": 3.926911459260109e-05,
+      "loss": 0.5766,
+      "step": 10416
+    },
+    {
+      "epoch": 0.0766042474404451,
+      "grad_norm": 1.001882553100586,
+      "learning_rate": 3.920007509089102e-05,
+      "loss": 0.5696,
+      "step": 10447
+    },
+    {
+      "epoch": 0.07683155974739003,
+      "grad_norm": 1.0127209424972534,
+      "learning_rate": 3.913087534326357e-05,
+      "loss": 0.5754,
+      "step": 10478
+    },
+    {
+      "epoch": 0.07705887205433497,
+      "grad_norm": 1.014377474784851,
+      "learning_rate": 3.9061516130628475e-05,
+      "loss": 0.5907,
+      "step": 10509
+    },
+    {
+      "epoch": 0.07728618436127992,
+      "grad_norm": 1.4645588397979736,
+      "learning_rate": 3.8991998235695025e-05,
+      "loss": 0.5736,
+      "step": 10540
+    },
+    {
+      "epoch": 0.07751349666822485,
+      "grad_norm": 1.0554478168487549,
+      "learning_rate": 3.8922322442963224e-05,
+      "loss": 0.5598,
+      "step": 10571
+    },
+    {
+      "epoch": 0.0777408089751698,
+      "grad_norm": 1.028587818145752,
+      "learning_rate": 3.885248953871491e-05,
+      "loss": 0.5631,
+      "step": 10602
+    },
+    {
+      "epoch": 0.07796812128211474,
+      "grad_norm": 1.0219361782073975,
+      "learning_rate": 3.8782500311004915e-05,
+      "loss": 0.5662,
+      "step": 10633
+    },
+    {
+      "epoch": 0.07819543358905968,
+      "grad_norm": 0.9541478753089905,
+      "learning_rate": 3.871235554965218e-05,
+      "loss": 0.5693,
+      "step": 10664
+    },
+    {
+      "epoch": 0.07842274589600462,
+      "grad_norm": 1.037392258644104,
+      "learning_rate": 3.864205604623078e-05,
+      "loss": 0.5796,
+      "step": 10695
+    },
+    {
+      "epoch": 0.07865005820294955,
+      "grad_norm": 1.0335135459899902,
+      "learning_rate": 3.857160259406107e-05,
+      "loss": 0.5736,
+      "step": 10726
+    },
+    {
+      "epoch": 0.0788773705098945,
+      "grad_norm": 1.0032522678375244,
+      "learning_rate": 3.8500995988200674e-05,
+      "loss": 0.5707,
+      "step": 10757
+    },
+    {
+      "epoch": 0.07910468281683944,
+      "grad_norm": 1.036737084388733,
+      "learning_rate": 3.843023702543556e-05,
+      "loss": 0.5626,
+      "step": 10788
+    },
+    {
+      "epoch": 0.07933199512378439,
+      "grad_norm": 1.0942667722702026,
+      "learning_rate": 3.8359326504270984e-05,
+      "loss": 0.5754,
+      "step": 10819
+    },
+    {
+      "epoch": 0.07955930743072932,
+      "grad_norm": 1.0547548532485962,
+      "learning_rate": 3.828826522492255e-05,
+      "loss": 0.5799,
+      "step": 10850
+    },
+    {
+      "epoch": 0.07978661973767427,
+      "grad_norm": 1.096701741218567,
+      "learning_rate": 3.821705398930713e-05,
+      "loss": 0.5826,
+      "step": 10881
+    },
+    {
+      "epoch": 0.08001393204461921,
+      "grad_norm": 0.8688652515411377,
+      "learning_rate": 3.814569360103385e-05,
+      "loss": 0.5638,
+      "step": 10912
+    },
+    {
+      "epoch": 0.08024124435156414,
+      "grad_norm": 0.9706131219863892,
+      "learning_rate": 3.807418486539499e-05,
+      "loss": 0.5667,
+      "step": 10943
+    },
+    {
+      "epoch": 0.08046855665850909,
+      "grad_norm": 0.9783776998519897,
+      "learning_rate": 3.80025285893569e-05,
+      "loss": 0.5694,
+      "step": 10974
+    },
+    {
+      "epoch": 0.08069586896545403,
+      "grad_norm": 0.9906594753265381,
+      "learning_rate": 3.793072558155093e-05,
+      "loss": 0.567,
+      "step": 11005
+    },
+    {
+      "epoch": 0.08092318127239898,
+      "grad_norm": 0.9372761845588684,
+      "learning_rate": 3.785877665226426e-05,
+      "loss": 0.5788,
+      "step": 11036
+    },
+    {
+      "epoch": 0.08115049357934391,
+      "grad_norm": 1.0891989469528198,
+      "learning_rate": 3.778668261343079e-05,
+      "loss": 0.5834,
+      "step": 11067
+    },
+    {
+      "epoch": 0.08137780588628885,
+      "grad_norm": 1.0318610668182373,
+      "learning_rate": 3.771444427862192e-05,
+      "loss": 0.5797,
+      "step": 11098
+    },
+    {
+      "epoch": 0.0816051181932338,
+      "grad_norm": 1.044351577758789,
+      "learning_rate": 3.7642062463037465e-05,
+      "loss": 0.5733,
+      "step": 11129
+    },
+    {
+      "epoch": 0.08183243050017873,
+      "grad_norm": 1.038954734802246,
+      "learning_rate": 3.7569537983496373e-05,
+      "loss": 0.5652,
+      "step": 11160
+    },
+    {
+      "epoch": 0.08205974280712368,
+      "grad_norm": 0.9230037927627563,
+      "learning_rate": 3.749687165842753e-05,
+      "loss": 0.5682,
+      "step": 11191
+    },
+    {
+      "epoch": 0.08228705511406861,
+      "grad_norm": 1.084397554397583,
+      "learning_rate": 3.7424064307860536e-05,
+      "loss": 0.5658,
+      "step": 11222
+    },
+    {
+      "epoch": 0.08251436742101356,
+      "grad_norm": 0.9791586399078369,
+      "learning_rate": 3.735111675341645e-05,
+      "loss": 0.5664,
+      "step": 11253
+    },
+    {
+      "epoch": 0.0827416797279585,
+      "grad_norm": 1.0730615854263306,
+      "learning_rate": 3.7278029818298524e-05,
+      "loss": 0.5684,
+      "step": 11284
+    },
+    {
+      "epoch": 0.08296899203490343,
+      "grad_norm": 1.0292216539382935,
+      "learning_rate": 3.720480432728287e-05,
+      "loss": 0.5683,
+      "step": 11315
+    },
+    {
+      "epoch": 0.08319630434184838,
+      "grad_norm": 1.1781240701675415,
+      "learning_rate": 3.71314411067092e-05,
+      "loss": 0.5742,
+      "step": 11346
+    },
+    {
+      "epoch": 0.08342361664879332,
+      "grad_norm": 1.2368265390396118,
+      "learning_rate": 3.70579409844715e-05,
+      "loss": 0.5662,
+      "step": 11377
+    },
+    {
+      "epoch": 0.08365092895573827,
+      "grad_norm": 0.9175316691398621,
+      "learning_rate": 3.698430479000865e-05,
+      "loss": 0.5767,
+      "step": 11408
+    },
+    {
+      "epoch": 0.0838782412626832,
+      "grad_norm": 1.6277068853378296,
+      "learning_rate": 3.691053335429509e-05,
+      "loss": 0.5588,
+      "step": 11439
+    },
+    {
+      "epoch": 0.08410555356962814,
+      "grad_norm": 1.0639629364013672,
+      "learning_rate": 3.683662750983147e-05,
+      "loss": 0.5771,
+      "step": 11470
+    },
+    {
+      "epoch": 0.08433286587657308,
+      "grad_norm": 0.9868754744529724,
+      "learning_rate": 3.676258809063518e-05,
+      "loss": 0.5598,
+      "step": 11501
+    },
+    {
+      "epoch": 0.08456017818351802,
+      "grad_norm": 0.8968964219093323,
+      "learning_rate": 3.6688415932231004e-05,
+      "loss": 0.5655,
+      "step": 11532
+    },
+    {
+      "epoch": 0.08478749049046297,
+      "grad_norm": 0.9365272521972656,
+      "learning_rate": 3.661411187164166e-05,
+      "loss": 0.5665,
+      "step": 11563
+    },
+    {
+      "epoch": 0.0850148027974079,
+      "grad_norm": 1.0714424848556519,
+      "learning_rate": 3.65396767473784e-05,
+      "loss": 0.5822,
+      "step": 11594
+    },
+    {
+      "epoch": 0.08524211510435285,
+      "grad_norm": 0.8779135346412659,
+      "learning_rate": 3.6465111399431465e-05,
+      "loss": 0.5606,
+      "step": 11625
+    },
+    {
+      "epoch": 0.08546942741129779,
+      "grad_norm": 0.9442996382713318,
+      "learning_rate": 3.6390416669260674e-05,
+      "loss": 0.568,
+      "step": 11656
+    },
+    {
+      "epoch": 0.08569673971824272,
+      "grad_norm": 0.9862335324287415,
+      "learning_rate": 3.63155933997859e-05,
+      "loss": 0.564,
+      "step": 11687
+    },
+    {
+      "epoch": 0.08592405202518767,
+      "grad_norm": 0.9895716309547424,
+      "learning_rate": 3.624064243537758e-05,
+      "loss": 0.5687,
+      "step": 11718
+    },
+    {
+      "epoch": 0.08615136433213261,
+      "grad_norm": 1.5118094682693481,
+      "learning_rate": 3.616556462184716e-05,
+      "loss": 0.5628,
+      "step": 11749
+    },
+    {
+      "epoch": 0.08637867663907756,
+      "grad_norm": 1.0005770921707153,
+      "learning_rate": 3.609036080643755e-05,
+      "loss": 0.5655,
+      "step": 11780
+    },
+    {
+      "epoch": 0.08660598894602249,
+      "grad_norm": 1.034129023551941,
+      "learning_rate": 3.60150318378136e-05,
+      "loss": 0.5741,
+      "step": 11811
+    },
+    {
+      "epoch": 0.08683330125296744,
+      "grad_norm": 0.9524271488189697,
+      "learning_rate": 3.5939578566052465e-05,
+      "loss": 0.5608,
+      "step": 11842
+    },
+    {
+      "epoch": 0.08706061355991238,
+      "grad_norm": 1.0711724758148193,
+      "learning_rate": 3.586400184263408e-05,
+      "loss": 0.5682,
+      "step": 11873
+    },
+    {
+      "epoch": 0.08728792586685731,
+      "grad_norm": 0.9183467030525208,
+      "learning_rate": 3.578830252043148e-05,
+      "loss": 0.5593,
+      "step": 11904
+    },
+    {
+      "epoch": 0.08751523817380226,
+      "grad_norm": 0.8957611918449402,
+      "learning_rate": 3.571248145370125e-05,
+      "loss": 0.5783,
+      "step": 11935
+    },
+    {
+      "epoch": 0.0877425504807472,
+      "grad_norm": 1.1284699440002441,
+      "learning_rate": 3.5636539498073794e-05,
+      "loss": 0.5888,
+      "step": 11966
+    },
+    {
+      "epoch": 0.08796986278769214,
+      "grad_norm": 1.074945330619812,
+      "learning_rate": 3.556047751054378e-05,
+      "loss": 0.5605,
+      "step": 11997
+    },
+    {
+      "epoch": 0.08819717509463708,
+      "grad_norm": 0.9872075319290161,
+      "learning_rate": 3.548429634946039e-05,
+      "loss": 0.562,
+      "step": 12028
+    },
+    {
+      "epoch": 0.08842448740158201,
+      "grad_norm": 0.9612171649932861,
+      "learning_rate": 3.540799687451768e-05,
+      "loss": 0.5686,
+      "step": 12059
+    },
+    {
+      "epoch": 0.08865179970852696,
+      "grad_norm": 1.058897614479065,
+      "learning_rate": 3.533157994674485e-05,
+      "loss": 0.5684,
+      "step": 12090
+    },
+    {
+      "epoch": 0.0888791120154719,
+      "grad_norm": 0.9676720499992371,
+      "learning_rate": 3.5255046428496546e-05,
+      "loss": 0.57,
+      "step": 12121
+    },
+    {
+      "epoch": 0.08910642432241685,
+      "grad_norm": 0.9120715856552124,
+      "learning_rate": 3.517839718344311e-05,
+      "loss": 0.5502,
+      "step": 12152
+    },
+    {
+      "epoch": 0.08933373662936178,
+      "grad_norm": 1.1131491661071777,
+      "learning_rate": 3.510163307656086e-05,
+      "loss": 0.5601,
+      "step": 12183
+    },
+    {
+      "epoch": 0.08956104893630673,
+      "grad_norm": 0.9530042409896851,
+      "learning_rate": 3.5024754974122324e-05,
+      "loss": 0.5577,
+      "step": 12214
+    },
+    {
+      "epoch": 0.08978836124325167,
+      "grad_norm": 0.8555998802185059,
+      "learning_rate": 3.494776374368643e-05,
+      "loss": 0.5554,
+      "step": 12245
+    },
+    {
+      "epoch": 0.0900156735501966,
+      "grad_norm": 0.9992236495018005,
+      "learning_rate": 3.4870660254088724e-05,
+      "loss": 0.5596,
+      "step": 12276
+    },
+    {
+      "epoch": 0.09024298585714155,
+      "grad_norm": 0.9591524004936218,
+      "learning_rate": 3.479344537543164e-05,
+      "loss": 0.5711,
+      "step": 12307
+    },
+    {
+      "epoch": 0.09047029816408649,
+      "grad_norm": 0.9413798451423645,
+      "learning_rate": 3.4716119979074565e-05,
+      "loss": 0.5654,
+      "step": 12338
+    },
+    {
+      "epoch": 0.09069761047103143,
+      "grad_norm": 0.9504027962684631,
+      "learning_rate": 3.463868493762412e-05,
+      "loss": 0.5683,
+      "step": 12369
+    },
+    {
+      "epoch": 0.09092492277797637,
+      "grad_norm": 0.9432477951049805,
+      "learning_rate": 3.456114112492418e-05,
+      "loss": 0.5645,
+      "step": 12400
+    },
+    {
+      "epoch": 0.09115223508492132,
+      "grad_norm": 0.9719948768615723,
+      "learning_rate": 3.4483489416046164e-05,
+      "loss": 0.5725,
+      "step": 12431
+    },
+    {
+      "epoch": 0.09137954739186625,
+      "grad_norm": 1.2095508575439453,
+      "learning_rate": 3.440573068727905e-05,
+      "loss": 0.5611,
+      "step": 12462
+    },
+    {
+      "epoch": 0.09160685969881119,
+      "grad_norm": 1.009135127067566,
+      "learning_rate": 3.4327865816119495e-05,
+      "loss": 0.5815,
+      "step": 12493
+    },
+    {
+      "epoch": 0.09183417200575614,
+      "grad_norm": 1.0403270721435547,
+      "learning_rate": 3.4249895681262025e-05,
+      "loss": 0.5788,
+      "step": 12524
+    },
+    {
+      "epoch": 0.09206148431270107,
+      "grad_norm": 1.0324249267578125,
+      "learning_rate": 3.417182116258899e-05,
+      "loss": 0.5621,
+      "step": 12555
+    },
+    {
+      "epoch": 0.09228879661964602,
+      "grad_norm": 1.0692311525344849,
+      "learning_rate": 3.409364314116074e-05,
+      "loss": 0.5675,
+      "step": 12586
+    },
+    {
+      "epoch": 0.09251610892659096,
+      "grad_norm": 0.9427818059921265,
+      "learning_rate": 3.401536249920559e-05,
+      "loss": 0.5573,
+      "step": 12617
+    },
+    {
+      "epoch": 0.09274342123353589,
+      "grad_norm": 0.93168705701828,
+      "learning_rate": 3.393698012010998e-05,
+      "loss": 0.561,
+      "step": 12648
+    },
+    {
+      "epoch": 0.09297073354048084,
+      "grad_norm": 1.0959157943725586,
+      "learning_rate": 3.385849688840839e-05,
+      "loss": 0.5587,
+      "step": 12679
+    },
+    {
+      "epoch": 0.09319804584742578,
+      "grad_norm": 0.9450280070304871,
+      "learning_rate": 3.3779913689773414e-05,
+      "loss": 0.5579,
+      "step": 12710
+    },
+    {
+      "epoch": 0.09342535815437072,
+      "grad_norm": 1.1094130277633667,
+      "learning_rate": 3.370123141100578e-05,
+      "loss": 0.5652,
+      "step": 12741
+    },
+    {
+      "epoch": 0.09365267046131566,
+      "grad_norm": 0.9264737367630005,
+      "learning_rate": 3.3622450940024305e-05,
+      "loss": 0.5633,
+      "step": 12772
+    },
+    {
+      "epoch": 0.09387998276826061,
+      "grad_norm": 0.9424936771392822,
+      "learning_rate": 3.35435731658559e-05,
+      "loss": 0.5519,
+      "step": 12803
+    },
+    {
+      "epoch": 0.09410729507520554,
+      "grad_norm": 0.9607046246528625,
+      "learning_rate": 3.346459897862552e-05,
+      "loss": 0.5628,
+      "step": 12834
+    },
+    {
+      "epoch": 0.09433460738215048,
+      "grad_norm": 0.8366544842720032,
+      "learning_rate": 3.338552926954613e-05,
+      "loss": 0.5667,
+      "step": 12865
+    },
+    {
+      "epoch": 0.09456191968909543,
+      "grad_norm": 0.9299005270004272,
+      "learning_rate": 3.330636493090868e-05,
+      "loss": 0.5703,
+      "step": 12896
+    },
+    {
+      "epoch": 0.09478923199604036,
+      "grad_norm": 1.15682053565979,
+      "learning_rate": 3.322710685607193e-05,
+      "loss": 0.5692,
+      "step": 12927
+    },
+    {
+      "epoch": 0.09501654430298531,
+      "grad_norm": 0.9281063079833984,
+      "learning_rate": 3.314775593945251e-05,
+      "loss": 0.5481,
+      "step": 12958
+    },
+    {
+      "epoch": 0.09524385660993025,
+      "grad_norm": 0.8622137308120728,
+      "learning_rate": 3.3068313076514714e-05,
+      "loss": 0.5533,
+      "step": 12989
+    },
+    {
+      "epoch": 0.0954711689168752,
+      "grad_norm": 0.9851906299591064,
+      "learning_rate": 3.298877916376047e-05,
+      "loss": 0.5472,
+      "step": 13020
+    },
+    {
+      "epoch": 0.09569848122382013,
+      "grad_norm": 1.0084940195083618,
+      "learning_rate": 3.290915509871915e-05,
+      "loss": 0.5532,
+      "step": 13051
+    },
+    {
+      "epoch": 0.09592579353076507,
+      "grad_norm": 1.7121292352676392,
+      "learning_rate": 3.282944177993753e-05,
+      "loss": 0.5828,
+      "step": 13082
+    },
+    {
+      "epoch": 0.09615310583771002,
+      "grad_norm": 0.987533450126648,
+      "learning_rate": 3.274964010696957e-05,
+      "loss": 0.5577,
+      "step": 13113
+    },
+    {
+      "epoch": 0.09638041814465495,
+      "grad_norm": 1.0349764823913574,
+      "learning_rate": 3.266975098036629e-05,
+      "loss": 0.5631,
+      "step": 13144
+    },
+    {
+      "epoch": 0.0966077304515999,
+      "grad_norm": 0.9670729041099548,
+      "learning_rate": 3.258977530166562e-05,
+      "loss": 0.557,
+      "step": 13175
+    },
+    {
+      "epoch": 0.09683504275854483,
+      "grad_norm": 1.0308313369750977,
+      "learning_rate": 3.250971397338227e-05,
+      "loss": 0.5643,
+      "step": 13206
+    },
+    {
+      "epoch": 0.09706235506548977,
+      "grad_norm": 0.9608160853385925,
+      "learning_rate": 3.2429567898997404e-05,
+      "loss": 0.561,
+      "step": 13237
+    },
+    {
+      "epoch": 0.09728966737243472,
+      "grad_norm": 1.045312523841858,
+      "learning_rate": 3.234933798294859e-05,
+      "loss": 0.5637,
+      "step": 13268
+    },
+    {
+      "epoch": 0.09751697967937965,
+      "grad_norm": 1.0054733753204346,
+      "learning_rate": 3.2269025130619535e-05,
+      "loss": 0.5534,
+      "step": 13299
+    },
+    {
+      "epoch": 0.0977442919863246,
+      "grad_norm": 1.1498805284500122,
+      "learning_rate": 3.218863024832985e-05,
+      "loss": 0.5568,
+      "step": 13330
+    },
+    {
+      "epoch": 0.09797160429326954,
+      "grad_norm": 1.0187801122665405,
+      "learning_rate": 3.2108154243324864e-05,
+      "loss": 0.5442,
+      "step": 13361
+    },
+    {
+      "epoch": 0.09819891660021449,
+      "grad_norm": 1.0210034847259521,
+      "learning_rate": 3.2027598023765345e-05,
+      "loss": 0.5667,
+      "step": 13392
+    },
+    {
+      "epoch": 0.09842622890715942,
+      "grad_norm": 1.0303422212600708,
+      "learning_rate": 3.194696249871729e-05,
+      "loss": 0.5575,
+      "step": 13423
+    },
+    {
+      "epoch": 0.09865354121410436,
+      "grad_norm": 1.1709095239639282,
+      "learning_rate": 3.186624857814164e-05,
+      "loss": 0.5608,
+      "step": 13454
+    },
+    {
+      "epoch": 0.0988808535210493,
+      "grad_norm": 1.101671814918518,
+      "learning_rate": 3.178545717288401e-05,
+      "loss": 0.553,
+      "step": 13485
+    },
+    {
+      "epoch": 0.09910816582799424,
+      "grad_norm": 0.8766924142837524,
+      "learning_rate": 3.170458919466444e-05,
+      "loss": 0.5621,
+      "step": 13516
+    },
+    {
+      "epoch": 0.09933547813493919,
+      "grad_norm": 0.8552247881889343,
+      "learning_rate": 3.1623645556067063e-05,
+      "loss": 0.5524,
+      "step": 13547
+    },
+    {
+      "epoch": 0.09956279044188412,
+      "grad_norm": 0.8928302526473999,
+      "learning_rate": 3.154262717052985e-05,
+      "loss": 0.5591,
+      "step": 13578
+    },
+    {
+      "epoch": 0.09979010274882906,
+      "grad_norm": 0.9657186269760132,
+      "learning_rate": 3.146153495233426e-05,
+      "loss": 0.5549,
+      "step": 13609
+    },
+    {
+      "epoch": 0.10001741505577401,
+      "grad_norm": 0.9948280453681946,
+      "learning_rate": 3.1380369816594944e-05,
+      "loss": 0.561,
+      "step": 13640
+    },
+    {
+      "epoch": 0.10024472736271894,
+      "grad_norm": 1.6007908582687378,
+      "learning_rate": 3.129913267924946e-05,
+      "loss": 0.5551,
+      "step": 13671
+    },
+    {
+      "epoch": 0.10047203966966389,
+      "grad_norm": 1.0863940715789795,
+      "learning_rate": 3.121782445704782e-05,
+      "loss": 0.5623,
+      "step": 13702
+    },
+    {
+      "epoch": 0.10069935197660883,
+      "grad_norm": 1.03696870803833,
+      "learning_rate": 3.11364460675423e-05,
+      "loss": 0.5597,
+      "step": 13733
+    },
+    {
+      "epoch": 0.10092666428355378,
+      "grad_norm": 0.9754369854927063,
+      "learning_rate": 3.1054998429076934e-05,
+      "loss": 0.5624,
+      "step": 13764
+    },
+    {
+      "epoch": 0.10115397659049871,
+      "grad_norm": 0.9959014654159546,
+      "learning_rate": 3.097348246077728e-05,
+      "loss": 0.5493,
+      "step": 13795
+    },
+    {
+      "epoch": 0.10138128889744365,
+      "grad_norm": 0.9870645403862,
+      "learning_rate": 3.0891899082539924e-05,
+      "loss": 0.5564,
+      "step": 13826
+    },
+    {
+      "epoch": 0.1016086012043886,
+      "grad_norm": 0.8938599228858948,
+      "learning_rate": 3.0810249215022233e-05,
+      "loss": 0.5516,
+      "step": 13857
+    },
+    {
+      "epoch": 0.10183591351133353,
+      "grad_norm": 0.9292534589767456,
+      "learning_rate": 3.0728533779631865e-05,
+      "loss": 0.5559,
+      "step": 13888
+    },
+    {
+      "epoch": 0.10206322581827848,
+      "grad_norm": 0.9075298309326172,
+      "learning_rate": 3.064675369851637e-05,
+      "loss": 0.5581,
+      "step": 13919
+    },
+    {
+      "epoch": 0.10229053812522342,
+      "grad_norm": 0.8756698369979858,
+      "learning_rate": 3.056490989455289e-05,
+      "loss": 0.5457,
+      "step": 13950
+    },
+    {
+      "epoch": 0.10251785043216836,
+      "grad_norm": 0.8829846978187561,
+      "learning_rate": 3.0483003291337596e-05,
+      "loss": 0.5574,
+      "step": 13981
+    },
+    {
+      "epoch": 0.1027451627391133,
+      "grad_norm": 0.9124001860618591,
+      "learning_rate": 3.040103481317539e-05,
+      "loss": 0.5544,
+      "step": 14012
+    },
+    {
+      "epoch": 0.10297247504605823,
+      "grad_norm": 0.905826985836029,
+      "learning_rate": 3.03190053850694e-05,
+      "loss": 0.5564,
+      "step": 14043
+    },
+    {
+      "epoch": 0.10319978735300318,
+      "grad_norm": 0.9436296820640564,
+      "learning_rate": 3.0236915932710573e-05,
+      "loss": 0.5579,
+      "step": 14074
+    },
+    {
+      "epoch": 0.10342709965994812,
+      "grad_norm": 1.052493691444397,
+      "learning_rate": 3.0154767382467232e-05,
+      "loss": 0.5562,
+      "step": 14105
+    },
+    {
+      "epoch": 0.10365441196689307,
+      "grad_norm": 0.9711372256278992,
+      "learning_rate": 3.0072560661374582e-05,
+      "loss": 0.5507,
+      "step": 14136
+    },
+    {
+      "epoch": 0.103881724273838,
+      "grad_norm": 1.1050643920898438,
+      "learning_rate": 2.999029669712431e-05,
+      "loss": 0.5579,
+      "step": 14167
+    },
+    {
+      "epoch": 0.10410903658078294,
+      "grad_norm": 0.9102250337600708,
+      "learning_rate": 2.990797641805408e-05,
+      "loss": 0.5655,
+      "step": 14198
+    },
+    {
+      "epoch": 0.10433634888772789,
+      "grad_norm": 0.9284120798110962,
+      "learning_rate": 2.982560075313704e-05,
+      "loss": 0.558,
+      "step": 14229
+    },
+    {
+      "epoch": 0.10456366119467282,
+      "grad_norm": 0.8311032652854919,
+      "learning_rate": 2.9743170631971368e-05,
+      "loss": 0.5529,
+      "step": 14260
+    },
+    {
+      "epoch": 0.10479097350161777,
+      "grad_norm": 0.8547165989875793,
+      "learning_rate": 2.9660686984769792e-05,
+      "loss": 0.552,
+      "step": 14291
+    },
+    {
+      "epoch": 0.1050182858085627,
+      "grad_norm": 0.883914053440094,
+      "learning_rate": 2.9578150742349047e-05,
+      "loss": 0.5671,
+      "step": 14322
+    },
+    {
+      "epoch": 0.10524559811550765,
+      "grad_norm": 0.9841529726982117,
+      "learning_rate": 2.949556283611942e-05,
+      "loss": 0.5666,
+      "step": 14353
+    },
+    {
+      "epoch": 0.10547291042245259,
+      "grad_norm": 0.9871898293495178,
+      "learning_rate": 2.9412924198074206e-05,
+      "loss": 0.5467,
+      "step": 14384
+    },
+    {
+      "epoch": 0.10570022272939752,
+      "grad_norm": 1.0510107278823853,
+      "learning_rate": 2.9330235760779208e-05,
+      "loss": 0.559,
+      "step": 14415
+    },
+    {
+      "epoch": 0.10592753503634247,
+      "grad_norm": 0.9736201763153076,
+      "learning_rate": 2.9247498457362188e-05,
+      "loss": 0.5506,
+      "step": 14446
+    },
+    {
+      "epoch": 0.10615484734328741,
+      "grad_norm": 0.9611804485321045,
+      "learning_rate": 2.9164713221502373e-05,
+      "loss": 0.5613,
+      "step": 14477
+    },
+    {
+      "epoch": 0.10638215965023236,
+      "grad_norm": 0.9581224918365479,
+      "learning_rate": 2.9081880987419912e-05,
+      "loss": 0.5503,
+      "step": 14508
+    },
+    {
+      "epoch": 0.1066094719571773,
+      "grad_norm": 0.9223282933235168,
+      "learning_rate": 2.8999002689865296e-05,
+      "loss": 0.5681,
+      "step": 14539
+    },
+    {
+      "epoch": 0.10683678426412224,
+      "grad_norm": 0.9399698376655579,
+      "learning_rate": 2.8916079264108852e-05,
+      "loss": 0.5499,
+      "step": 14570
+    },
+    {
+      "epoch": 0.10706409657106718,
+      "grad_norm": 0.8871161937713623,
+      "learning_rate": 2.883311164593017e-05,
+      "loss": 0.561,
+      "step": 14601
+    },
+    {
+      "epoch": 0.10729140887801211,
+      "grad_norm": 0.9508283734321594,
+      "learning_rate": 2.875010077160754e-05,
+      "loss": 0.5457,
+      "step": 14632
+    },
+    {
+      "epoch": 0.10751872118495706,
+      "grad_norm": 0.9116697311401367,
+      "learning_rate": 2.866704757790741e-05,
+      "loss": 0.5644,
+      "step": 14663
+    },
+    {
+      "epoch": 0.107746033491902,
+      "grad_norm": 1.266608715057373,
+      "learning_rate": 2.858395300207376e-05,
+      "loss": 0.5535,
+      "step": 14694
+    },
+    {
+      "epoch": 0.10797334579884695,
+      "grad_norm": 0.9145412445068359,
+      "learning_rate": 2.8500817981817607e-05,
+      "loss": 0.5633,
+      "step": 14725
+    },
+    {
+      "epoch": 0.10820065810579188,
+      "grad_norm": 1.3427720069885254,
+      "learning_rate": 2.8417643455306336e-05,
+      "loss": 0.5524,
+      "step": 14756
+    },
+    {
+      "epoch": 0.10842797041273682,
+      "grad_norm": 0.8843008279800415,
+      "learning_rate": 2.8334430361153185e-05,
+      "loss": 0.5517,
+      "step": 14787
+    },
+    {
+      "epoch": 0.10865528271968176,
+      "grad_norm": 0.9420852065086365,
+      "learning_rate": 2.8251179638406612e-05,
+      "loss": 0.5491,
+      "step": 14818
+    },
+    {
+      "epoch": 0.1088825950266267,
+      "grad_norm": 0.9772655367851257,
+      "learning_rate": 2.8167892226539704e-05,
+      "loss": 0.549,
+      "step": 14849
+    },
+    {
+      "epoch": 0.10910990733357165,
+      "grad_norm": 0.957847535610199,
+      "learning_rate": 2.8084569065439588e-05,
+      "loss": 0.5449,
+      "step": 14880
+    },
+    {
+      "epoch": 0.10933721964051658,
+      "grad_norm": 0.9752157330513,
+      "learning_rate": 2.8001211095396807e-05,
+      "loss": 0.5534,
+      "step": 14911
+    },
+    {
+      "epoch": 0.10956453194746153,
+      "grad_norm": 0.9377481341362,
+      "learning_rate": 2.791781925709473e-05,
+      "loss": 0.5547,
+      "step": 14942
+    },
+    {
+      "epoch": 0.10979184425440647,
+      "grad_norm": 1.05467689037323,
+      "learning_rate": 2.7834394491598908e-05,
+      "loss": 0.5479,
+      "step": 14973
+    },
+    {
+      "epoch": 0.1100191565613514,
+      "grad_norm": 0.8407652974128723,
+      "learning_rate": 2.7750937740346485e-05,
+      "loss": 0.5583,
+      "step": 15004
+    },
+    {
+      "epoch": 0.11024646886829635,
+      "grad_norm": 0.9867594838142395,
+      "learning_rate": 2.7667449945135564e-05,
+      "loss": 0.5586,
+      "step": 15035
+    },
+    {
+      "epoch": 0.11047378117524129,
+      "grad_norm": 0.8983747959136963,
+      "learning_rate": 2.7583932048114557e-05,
+      "loss": 0.5496,
+      "step": 15066
+    },
+    {
+      "epoch": 0.11070109348218624,
+      "grad_norm": 0.893854558467865,
+      "learning_rate": 2.7500384991771587e-05,
+      "loss": 0.5419,
+      "step": 15097
+    },
+    {
+      "epoch": 0.11092840578913117,
+      "grad_norm": 1.0112414360046387,
+      "learning_rate": 2.7416809718923825e-05,
+      "loss": 0.568,
+      "step": 15128
+    },
+    {
+      "epoch": 0.11115571809607612,
+      "grad_norm": 0.8664485216140747,
+      "learning_rate": 2.7333207172706864e-05,
+      "loss": 0.5594,
+      "step": 15159
+    },
+    {
+      "epoch": 0.11138303040302105,
+      "grad_norm": 1.0966203212738037,
+      "learning_rate": 2.7249578296564088e-05,
+      "loss": 0.5559,
+      "step": 15190
+    },
+    {
+      "epoch": 0.11161034270996599,
+      "grad_norm": 21.80364418029785,
+      "learning_rate": 2.7165924034235973e-05,
+      "loss": 0.5449,
+      "step": 15221
+    },
+    {
+      "epoch": 0.11183765501691094,
+      "grad_norm": 0.894828200340271,
+      "learning_rate": 2.708224532974953e-05,
+      "loss": 0.5417,
+      "step": 15252
+    }
+  ],
+  "logging_steps": 31,
+  "max_steps": 30517,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 7630,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.1258280394232955e+19,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-15260/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd8bfcc3c69ebb637d33f900cde1394317a7271d81acc241b712bca643c7bd92
+size 5432

checkpoint-22890/config.json

@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "meta-llama/Llama-3.1-8B",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.47.0.dev0",
+  "use_cache": true,
+  "vocab_size": 128256
+}

checkpoint-22890/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.47.0.dev0"
+}

checkpoint-22890/model-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:259488925821336e28bbe02984d6d2e899981ae59adfc8563da28b4ad1e26ec7
+size 4886466168

checkpoint-22890/model-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0901d2e156c6ec7a375c3e90ae5183970af0445d3dc0e53c031b45673775dd64
+size 4832007448

checkpoint-22890/model-00003-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afa0e22bcade9527b5aeb86066e139cc30979f8492c753ff3277e6189863ab97
+size 4999813112

checkpoint-22890/model-00004-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc4f5352f30ea036b0a4e24ba9b029dd94a2b30a716c81acc1c4a5e7bb47a042
+size 4999813128

checkpoint-22890/model-00005-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d9450b209df4cb026b6669e618fcdb6ff4f5bd0a711fb2e502afe0b95eb3ad7
+size 4832007496

checkpoint-22890/model-00006-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:088d42bc0dd718059f7cf6efea50815186cc2c0e0db714af6eadcd1a920aacc9
+size 4999813120

checkpoint-22890/model-00007-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4a674ea8b90cbefff6b375516b0d3a006dab1ca9da5496de129732c50c01cdd
+size 2571158184

checkpoint-22890/model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 32121044992
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00007-of-00007.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.norm.weight": "model-00007-of-00007.safetensors"
+  }
+}

checkpoint-22890/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8427681722c954d1c8e7974d85f3aa1be5489cbf365b499cedefafdde6eb12f9
+size 15385036334

checkpoint-22890/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:386fcc8cc1089aade9450d86fb239ea3483f455fd2d78d8378645feecfec9d69
+size 14244

checkpoint-22890/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9cceae98d85860224f83926a9c2d0797a9f4411abc8a933e66f23226e9ba794a
+size 1064

checkpoint-22890/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd8bfcc3c69ebb637d33f900cde1394317a7271d81acc241b712bca643c7bd92
+size 5432

checkpoint-30517/config.json

@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "meta-llama/Llama-3.1-8B",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.47.0.dev0",
+  "use_cache": true,
+  "vocab_size": 128256
+}

checkpoint-30517/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.47.0.dev0"
+}

checkpoint-30517/model-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:461b4b5044270c180f3c02f130f42438555977ba6309f0d3f951b93defc69cbc
+size 4886466168

checkpoint-30517/model-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0901d2e156c6ec7a375c3e90ae5183970af0445d3dc0e53c031b45673775dd64
+size 4832007448

checkpoint-30517/model-00003-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afa0e22bcade9527b5aeb86066e139cc30979f8492c753ff3277e6189863ab97
+size 4999813112

checkpoint-30517/model-00004-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc4f5352f30ea036b0a4e24ba9b029dd94a2b30a716c81acc1c4a5e7bb47a042
+size 4999813128

checkpoint-30517/model-00005-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d9450b209df4cb026b6669e618fcdb6ff4f5bd0a711fb2e502afe0b95eb3ad7
+size 4832007496

checkpoint-30517/model-00006-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f796c1b0e88c58067dfd1457cbd57e07bf0133db27ae48d680980d7f503a9102
+size 4999813120

checkpoint-30517/model-00007-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e6376cef06327af2706504cac14f4e38345a3a9d2879661a9b1bea38d646ed5
+size 2571158184

checkpoint-30517/model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 32121044992
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00007-of-00007.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.norm.weight": "model-00007-of-00007.safetensors"
+  }
+}

checkpoint-30517/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1178528f1e77ef0b3d535824732b9855454645e79e1241b468cef764a563f5fd
+size 15385036334

checkpoint-30517/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:386fcc8cc1089aade9450d86fb239ea3483f455fd2d78d8378645feecfec9d69
+size 14244

checkpoint-30517/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d91c597306f26392e8da0d33a09fdcca77cfdeb5ad42248045521772fa3d64e
+size 1064

checkpoint-30517/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd8bfcc3c69ebb637d33f900cde1394317a7271d81acc241b712bca643c7bd92
+size 5432

checkpoint-7630/config.json

@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "meta-llama/Llama-3.1-8B",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.47.0.dev0",
+  "use_cache": true,
+  "vocab_size": 128256
+}

checkpoint-7630/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.47.0.dev0"
+}

checkpoint-7630/model-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cdce03f723037529d4fa0577dc4cf2283e4010c37f8cfa83da02d0145fa88c9d
+size 4886466168

checkpoint-7630/model-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0901d2e156c6ec7a375c3e90ae5183970af0445d3dc0e53c031b45673775dd64
+size 4832007448