Upload folder using huggingface_hub

checkpoint-15260/config.json

@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "/scratch/dme-llm/ayamaguchi.dme-llm/models/Llama-3.1-8B-ta-madlad-mean/",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.47.0.dev0",
+  "use_cache": true,
+  "vocab_size": 138256
+}

checkpoint-15260/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.47.0.dev0"
+}

checkpoint-15260/model-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54ec306fc2d6bb341e6870d47f0d477a559ed5e6741a06b17606b1def0c3a9ba
+size 4983197184

checkpoint-15260/model-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b28e50e95c787e68fdf10680635ba27e6d440763a69744dc6ddf473f852231a
+size 4899116432

checkpoint-15260/model-00003-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afa0e22bcade9527b5aeb86066e139cc30979f8492c753ff3277e6189863ab97
+size 4999813112

checkpoint-15260/model-00004-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc4f5352f30ea036b0a4e24ba9b029dd94a2b30a716c81acc1c4a5e7bb47a042
+size 4999813128

checkpoint-15260/model-00005-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d9450b209df4cb026b6669e618fcdb6ff4f5bd0a711fb2e502afe0b95eb3ad7
+size 4832007496

checkpoint-15260/model-00006-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51ce415c7adf6aa7dbd52ec109bf29ecfa851e9c6672b19e413070f3c7a51f0f
+size 4999813120

checkpoint-15260/model-00007-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:027f87e50cd54a9da79594e7af4834bf046b29ddf7e49ce2035cdd5124275f55
+size 2734998184

checkpoint-15260/model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 32448724992
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00007-of-00007.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.norm.weight": "model-00007-of-00007.safetensors"
+  }
+}

checkpoint-15260/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86fdfb1d27df1bcbec5a036dfa150c4082d250833c9dce1b9828066a20e555d5
+size 16040396334

checkpoint-15260/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:386fcc8cc1089aade9450d86fb239ea3483f455fd2d78d8378645feecfec9d69
+size 14244

checkpoint-15260/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ce335347efaebf038b8915bc0a8d2d587a4a1aa08ad0e015b4bc7cc4fba634e
+size 1064

checkpoint-15260/trainer_state.json

@@ -0,0 +1,3477 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.5071789417708057,
+  "eval_steps": 500,
+  "global_step": 15260,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.001030311087476735,
+      "grad_norm": 69.72277069091797,
+      "learning_rate": 1.0157273918741808e-06,
+      "loss": 9.0805,
+      "step": 31
+    },
+    {
+      "epoch": 0.00206062217495347,
+      "grad_norm": 15.001359939575195,
+      "learning_rate": 2.0314547837483616e-06,
+      "loss": 7.144,
+      "step": 62
+    },
+    {
+      "epoch": 0.003090933262430205,
+      "grad_norm": 13.807100296020508,
+      "learning_rate": 3.0471821756225426e-06,
+      "loss": 5.836,
+      "step": 93
+    },
+    {
+      "epoch": 0.00412124434990694,
+      "grad_norm": 25.857120513916016,
+      "learning_rate": 4.062909567496723e-06,
+      "loss": 4.7766,
+      "step": 124
+    },
+    {
+      "epoch": 0.005151555437383675,
+      "grad_norm": 27.085594177246094,
+      "learning_rate": 5.078636959370905e-06,
+      "loss": 4.1908,
+      "step": 155
+    },
+    {
+      "epoch": 0.00618186652486041,
+      "grad_norm": 37.11306381225586,
+      "learning_rate": 6.094364351245085e-06,
+      "loss": 3.864,
+      "step": 186
+    },
+    {
+      "epoch": 0.007212177612337145,
+      "grad_norm": 14.88463020324707,
+      "learning_rate": 7.110091743119267e-06,
+      "loss": 3.6287,
+      "step": 217
+    },
+    {
+      "epoch": 0.00824248869981388,
+      "grad_norm": 20.41550064086914,
+      "learning_rate": 8.125819134993446e-06,
+      "loss": 3.4712,
+      "step": 248
+    },
+    {
+      "epoch": 0.009272799787290615,
+      "grad_norm": 22.324840545654297,
+      "learning_rate": 9.141546526867629e-06,
+      "loss": 3.2928,
+      "step": 279
+    },
+    {
+      "epoch": 0.01030311087476735,
+      "grad_norm": 14.003937721252441,
+      "learning_rate": 1.015727391874181e-05,
+      "loss": 3.212,
+      "step": 310
+    },
+    {
+      "epoch": 0.011333421962244085,
+      "grad_norm": 15.5790433883667,
+      "learning_rate": 1.117300131061599e-05,
+      "loss": 3.1004,
+      "step": 341
+    },
+    {
+      "epoch": 0.01236373304972082,
+      "grad_norm": 10.85480785369873,
+      "learning_rate": 1.218872870249017e-05,
+      "loss": 3.0225,
+      "step": 372
+    },
+    {
+      "epoch": 0.013394044137197554,
+      "grad_norm": 12.166501998901367,
+      "learning_rate": 1.3204456094364351e-05,
+      "loss": 2.9216,
+      "step": 403
+    },
+    {
+      "epoch": 0.01442435522467429,
+      "grad_norm": 8.72249984741211,
+      "learning_rate": 1.4220183486238533e-05,
+      "loss": 2.8482,
+      "step": 434
+    },
+    {
+      "epoch": 0.015454666312151024,
+      "grad_norm": 9.419731140136719,
+      "learning_rate": 1.5235910878112714e-05,
+      "loss": 2.7928,
+      "step": 465
+    },
+    {
+      "epoch": 0.01648497739962776,
+      "grad_norm": 8.698421478271484,
+      "learning_rate": 1.6251638269986893e-05,
+      "loss": 2.7455,
+      "step": 496
+    },
+    {
+      "epoch": 0.017515288487104493,
+      "grad_norm": 11.099336624145508,
+      "learning_rate": 1.7267365661861077e-05,
+      "loss": 2.6973,
+      "step": 527
+    },
+    {
+      "epoch": 0.01854559957458123,
+      "grad_norm": 6.776867389678955,
+      "learning_rate": 1.8283093053735257e-05,
+      "loss": 2.6434,
+      "step": 558
+    },
+    {
+      "epoch": 0.019575910662057962,
+      "grad_norm": 8.685785293579102,
+      "learning_rate": 1.9298820445609438e-05,
+      "loss": 2.5981,
+      "step": 589
+    },
+    {
+      "epoch": 0.0206062217495347,
+      "grad_norm": 7.274338245391846,
+      "learning_rate": 2.031454783748362e-05,
+      "loss": 2.5588,
+      "step": 620
+    },
+    {
+      "epoch": 0.021636532837011432,
+      "grad_norm": 8.879731178283691,
+      "learning_rate": 2.13302752293578e-05,
+      "loss": 2.5313,
+      "step": 651
+    },
+    {
+      "epoch": 0.02266684392448817,
+      "grad_norm": 7.826289176940918,
+      "learning_rate": 2.234600262123198e-05,
+      "loss": 2.4972,
+      "step": 682
+    },
+    {
+      "epoch": 0.023697155011964902,
+      "grad_norm": 6.979713439941406,
+      "learning_rate": 2.336173001310616e-05,
+      "loss": 2.454,
+      "step": 713
+    },
+    {
+      "epoch": 0.02472746609944164,
+      "grad_norm": 5.871748924255371,
+      "learning_rate": 2.437745740498034e-05,
+      "loss": 2.4213,
+      "step": 744
+    },
+    {
+      "epoch": 0.025757777186918372,
+      "grad_norm": 7.163940906524658,
+      "learning_rate": 2.5393184796854525e-05,
+      "loss": 2.3794,
+      "step": 775
+    },
+    {
+      "epoch": 0.02678808827439511,
+      "grad_norm": 6.183249473571777,
+      "learning_rate": 2.6408912188728702e-05,
+      "loss": 2.3387,
+      "step": 806
+    },
+    {
+      "epoch": 0.027818399361871842,
+      "grad_norm": 6.347677230834961,
+      "learning_rate": 2.7424639580602886e-05,
+      "loss": 2.3235,
+      "step": 837
+    },
+    {
+      "epoch": 0.02884871044934858,
+      "grad_norm": 6.384498596191406,
+      "learning_rate": 2.8440366972477066e-05,
+      "loss": 2.2983,
+      "step": 868
+    },
+    {
+      "epoch": 0.029879021536825312,
+      "grad_norm": 4.966649532318115,
+      "learning_rate": 2.9456094364351244e-05,
+      "loss": 2.2701,
+      "step": 899
+    },
+    {
+      "epoch": 0.03090933262430205,
+      "grad_norm": 4.778007984161377,
+      "learning_rate": 3.0471821756225428e-05,
+      "loss": 2.2314,
+      "step": 930
+    },
+    {
+      "epoch": 0.031939643711778785,
+      "grad_norm": 4.620309352874756,
+      "learning_rate": 3.148754914809961e-05,
+      "loss": 2.2097,
+      "step": 961
+    },
+    {
+      "epoch": 0.03296995479925552,
+      "grad_norm": 5.562178134918213,
+      "learning_rate": 3.2503276539973785e-05,
+      "loss": 2.194,
+      "step": 992
+    },
+    {
+      "epoch": 0.03400026588673225,
+      "grad_norm": 4.52315092086792,
+      "learning_rate": 3.351900393184797e-05,
+      "loss": 2.1515,
+      "step": 1023
+    },
+    {
+      "epoch": 0.035030576974208985,
+      "grad_norm": 4.067371368408203,
+      "learning_rate": 3.453473132372215e-05,
+      "loss": 2.1247,
+      "step": 1054
+    },
+    {
+      "epoch": 0.036060888061685725,
+      "grad_norm": 4.2394633293151855,
+      "learning_rate": 3.555045871559633e-05,
+      "loss": 2.1139,
+      "step": 1085
+    },
+    {
+      "epoch": 0.03709119914916246,
+      "grad_norm": 3.944603204727173,
+      "learning_rate": 3.6566186107470514e-05,
+      "loss": 2.0886,
+      "step": 1116
+    },
+    {
+      "epoch": 0.03812151023663919,
+      "grad_norm": 4.198657989501953,
+      "learning_rate": 3.7581913499344695e-05,
+      "loss": 2.0925,
+      "step": 1147
+    },
+    {
+      "epoch": 0.039151821324115925,
+      "grad_norm": 3.8212976455688477,
+      "learning_rate": 3.8597640891218876e-05,
+      "loss": 2.0635,
+      "step": 1178
+    },
+    {
+      "epoch": 0.04018213241159266,
+      "grad_norm": 4.050039768218994,
+      "learning_rate": 3.9613368283093056e-05,
+      "loss": 2.0478,
+      "step": 1209
+    },
+    {
+      "epoch": 0.0412124434990694,
+      "grad_norm": 4.439040660858154,
+      "learning_rate": 4.062909567496724e-05,
+      "loss": 2.0299,
+      "step": 1240
+    },
+    {
+      "epoch": 0.04224275458654613,
+      "grad_norm": 13.521013259887695,
+      "learning_rate": 4.164482306684142e-05,
+      "loss": 2.035,
+      "step": 1271
+    },
+    {
+      "epoch": 0.043273065674022865,
+      "grad_norm": 3.7606613636016846,
+      "learning_rate": 4.26605504587156e-05,
+      "loss": 2.0048,
+      "step": 1302
+    },
+    {
+      "epoch": 0.0443033767614996,
+      "grad_norm": 3.348107099533081,
+      "learning_rate": 4.367627785058978e-05,
+      "loss": 1.9905,
+      "step": 1333
+    },
+    {
+      "epoch": 0.04533368784897634,
+      "grad_norm": 3.5753731727600098,
+      "learning_rate": 4.469200524246396e-05,
+      "loss": 1.9999,
+      "step": 1364
+    },
+    {
+      "epoch": 0.04636399893645307,
+      "grad_norm": 3.3659353256225586,
+      "learning_rate": 4.570773263433814e-05,
+      "loss": 1.9461,
+      "step": 1395
+    },
+    {
+      "epoch": 0.047394310023929805,
+      "grad_norm": 3.0083465576171875,
+      "learning_rate": 4.672346002621232e-05,
+      "loss": 1.9646,
+      "step": 1426
+    },
+    {
+      "epoch": 0.04842462111140654,
+      "grad_norm": 3.514509439468384,
+      "learning_rate": 4.77391874180865e-05,
+      "loss": 1.956,
+      "step": 1457
+    },
+    {
+      "epoch": 0.04945493219888328,
+      "grad_norm": 3.0360188484191895,
+      "learning_rate": 4.875491480996068e-05,
+      "loss": 1.9425,
+      "step": 1488
+    },
+    {
+      "epoch": 0.05048524328636001,
+      "grad_norm": 2.9522669315338135,
+      "learning_rate": 4.977064220183487e-05,
+      "loss": 1.9419,
+      "step": 1519
+    },
+    {
+      "epoch": 0.051515554373836744,
+      "grad_norm": 3.3253414630889893,
+      "learning_rate": 4.9999915451558777e-05,
+      "loss": 1.9203,
+      "step": 1550
+    },
+    {
+      "epoch": 0.05254586546131348,
+      "grad_norm": 3.055213689804077,
+      "learning_rate": 4.999955597496219e-05,
+      "loss": 1.9142,
+      "step": 1581
+    },
+    {
+      "epoch": 0.05357617654879022,
+      "grad_norm": 2.9514503479003906,
+      "learning_rate": 4.9998914381774255e-05,
+      "loss": 1.9091,
+      "step": 1612
+    },
+    {
+      "epoch": 0.05460648763626695,
+      "grad_norm": 3.1836583614349365,
+      "learning_rate": 4.999799067923527e-05,
+      "loss": 1.9116,
+      "step": 1643
+    },
+    {
+      "epoch": 0.055636798723743684,
+      "grad_norm": 2.536240816116333,
+      "learning_rate": 4.999678487776908e-05,
+      "loss": 1.869,
+      "step": 1674
+    },
+    {
+      "epoch": 0.05666710981122042,
+      "grad_norm": 3.1409473419189453,
+      "learning_rate": 4.9995296990983006e-05,
+      "loss": 1.8982,
+      "step": 1705
+    },
+    {
+      "epoch": 0.05769742089869716,
+      "grad_norm": 2.711731195449829,
+      "learning_rate": 4.999352703566763e-05,
+      "loss": 1.8783,
+      "step": 1736
+    },
+    {
+      "epoch": 0.05872773198617389,
+      "grad_norm": 2.7510972023010254,
+      "learning_rate": 4.999147503179668e-05,
+      "loss": 1.8597,
+      "step": 1767
+    },
+    {
+      "epoch": 0.059758043073650624,
+      "grad_norm": 2.4287936687469482,
+      "learning_rate": 4.998914100252672e-05,
+      "loss": 1.8469,
+      "step": 1798
+    },
+    {
+      "epoch": 0.06078835416112736,
+      "grad_norm": 2.4476819038391113,
+      "learning_rate": 4.998652497419696e-05,
+      "loss": 1.848,
+      "step": 1829
+    },
+    {
+      "epoch": 0.0618186652486041,
+      "grad_norm": 2.417661666870117,
+      "learning_rate": 4.9983626976328927e-05,
+      "loss": 1.8416,
+      "step": 1860
+    },
+    {
+      "epoch": 0.06284897633608083,
+      "grad_norm": 2.6575193405151367,
+      "learning_rate": 4.998044704162613e-05,
+      "loss": 1.8351,
+      "step": 1891
+    },
+    {
+      "epoch": 0.06387928742355757,
+      "grad_norm": 2.573779344558716,
+      "learning_rate": 4.9976985205973705e-05,
+      "loss": 1.8313,
+      "step": 1922
+    },
+    {
+      "epoch": 0.0649095985110343,
+      "grad_norm": 2.3768863677978516,
+      "learning_rate": 4.997324150843799e-05,
+      "loss": 1.8382,
+      "step": 1953
+    },
+    {
+      "epoch": 0.06593990959851104,
+      "grad_norm": 2.5353589057922363,
+      "learning_rate": 4.99692159912661e-05,
+      "loss": 1.8095,
+      "step": 1984
+    },
+    {
+      "epoch": 0.06697022068598776,
+      "grad_norm": 2.35386323928833,
+      "learning_rate": 4.996490869988546e-05,
+      "loss": 1.8073,
+      "step": 2015
+    },
+    {
+      "epoch": 0.0680005317734645,
+      "grad_norm": 2.4447522163391113,
+      "learning_rate": 4.996031968290326e-05,
+      "loss": 1.7974,
+      "step": 2046
+    },
+    {
+      "epoch": 0.06903084286094124,
+      "grad_norm": 2.2738473415374756,
+      "learning_rate": 4.995544899210594e-05,
+      "loss": 1.8211,
+      "step": 2077
+    },
+    {
+      "epoch": 0.07006115394841797,
+      "grad_norm": 2.458066701889038,
+      "learning_rate": 4.9950296682458583e-05,
+      "loss": 1.7735,
+      "step": 2108
+    },
+    {
+      "epoch": 0.07109146503589471,
+      "grad_norm": 2.1359920501708984,
+      "learning_rate": 4.994486281210429e-05,
+      "loss": 1.7879,
+      "step": 2139
+    },
+    {
+      "epoch": 0.07212177612337145,
+      "grad_norm": 2.254345178604126,
+      "learning_rate": 4.9939147442363566e-05,
+      "loss": 1.8024,
+      "step": 2170
+    },
+    {
+      "epoch": 0.07315208721084818,
+      "grad_norm": 2.3210208415985107,
+      "learning_rate": 4.9933150637733574e-05,
+      "loss": 1.7933,
+      "step": 2201
+    },
+    {
+      "epoch": 0.07418239829832492,
+      "grad_norm": 2.108074426651001,
+      "learning_rate": 4.992687246588743e-05,
+      "loss": 1.7686,
+      "step": 2232
+    },
+    {
+      "epoch": 0.07521270938580164,
+      "grad_norm": 2.657787322998047,
+      "learning_rate": 4.992031299767347e-05,
+      "loss": 1.7666,
+      "step": 2263
+    },
+    {
+      "epoch": 0.07624302047327838,
+      "grad_norm": 2.380366563796997,
+      "learning_rate": 4.9913472307114386e-05,
+      "loss": 1.7861,
+      "step": 2294
+    },
+    {
+      "epoch": 0.07727333156075512,
+      "grad_norm": 2.310838222503662,
+      "learning_rate": 4.9906350471406446e-05,
+      "loss": 1.7604,
+      "step": 2325
+    },
+    {
+      "epoch": 0.07830364264823185,
+      "grad_norm": 2.4452033042907715,
+      "learning_rate": 4.989894757091861e-05,
+      "loss": 1.7721,
+      "step": 2356
+    },
+    {
+      "epoch": 0.07933395373570859,
+      "grad_norm": 2.0178720951080322,
+      "learning_rate": 4.989126368919158e-05,
+      "loss": 1.7592,
+      "step": 2387
+    },
+    {
+      "epoch": 0.08036426482318532,
+      "grad_norm": 2.1606602668762207,
+      "learning_rate": 4.988329891293693e-05,
+      "loss": 1.7367,
+      "step": 2418
+    },
+    {
+      "epoch": 0.08139457591066206,
+      "grad_norm": 2.069802761077881,
+      "learning_rate": 4.987505333203608e-05,
+      "loss": 1.7472,
+      "step": 2449
+    },
+    {
+      "epoch": 0.0824248869981388,
+      "grad_norm": 2.306144952774048,
+      "learning_rate": 4.9866527039539276e-05,
+      "loss": 1.7427,
+      "step": 2480
+    },
+    {
+      "epoch": 0.08345519808561552,
+      "grad_norm": 2.233604907989502,
+      "learning_rate": 4.9857720131664594e-05,
+      "loss": 1.7375,
+      "step": 2511
+    },
+    {
+      "epoch": 0.08448550917309226,
+      "grad_norm": 2.3948349952697754,
+      "learning_rate": 4.9848632707796773e-05,
+      "loss": 1.7474,
+      "step": 2542
+    },
+    {
+      "epoch": 0.085515820260569,
+      "grad_norm": 2.074847459793091,
+      "learning_rate": 4.9839264870486155e-05,
+      "loss": 1.7441,
+      "step": 2573
+    },
+    {
+      "epoch": 0.08654613134804573,
+      "grad_norm": 2.044692277908325,
+      "learning_rate": 4.9829616725447526e-05,
+      "loss": 1.7372,
+      "step": 2604
+    },
+    {
+      "epoch": 0.08757644243552247,
+      "grad_norm": 2.0216033458709717,
+      "learning_rate": 4.981968838155888e-05,
+      "loss": 1.7294,
+      "step": 2635
+    },
+    {
+      "epoch": 0.0886067535229992,
+      "grad_norm": 1.936315894126892,
+      "learning_rate": 4.980947995086024e-05,
+      "loss": 1.7127,
+      "step": 2666
+    },
+    {
+      "epoch": 0.08963706461047594,
+      "grad_norm": 2.021522283554077,
+      "learning_rate": 4.979899154855234e-05,
+      "loss": 1.7288,
+      "step": 2697
+    },
+    {
+      "epoch": 0.09066737569795268,
+      "grad_norm": 2.198458433151245,
+      "learning_rate": 4.9788223292995386e-05,
+      "loss": 1.7302,
+      "step": 2728
+    },
+    {
+      "epoch": 0.0916976867854294,
+      "grad_norm": 1.9769110679626465,
+      "learning_rate": 4.977717530570768e-05,
+      "loss": 1.7228,
+      "step": 2759
+    },
+    {
+      "epoch": 0.09272799787290614,
+      "grad_norm": 3.0774645805358887,
+      "learning_rate": 4.976584771136425e-05,
+      "loss": 1.7348,
+      "step": 2790
+    },
+    {
+      "epoch": 0.09375830896038288,
+      "grad_norm": 2.3825507164001465,
+      "learning_rate": 4.975424063779547e-05,
+      "loss": 1.6971,
+      "step": 2821
+    },
+    {
+      "epoch": 0.09478862004785961,
+      "grad_norm": 2.003248691558838,
+      "learning_rate": 4.974235421598557e-05,
+      "loss": 1.705,
+      "step": 2852
+    },
+    {
+      "epoch": 0.09581893113533635,
+      "grad_norm": 1.95453679561615,
+      "learning_rate": 4.973018858007122e-05,
+      "loss": 1.6926,
+      "step": 2883
+    },
+    {
+      "epoch": 0.09684924222281308,
+      "grad_norm": 2.0026769638061523,
+      "learning_rate": 4.9717743867339963e-05,
+      "loss": 1.6989,
+      "step": 2914
+    },
+    {
+      "epoch": 0.09787955331028982,
+      "grad_norm": 2.098222494125366,
+      "learning_rate": 4.9705020218228695e-05,
+      "loss": 1.7013,
+      "step": 2945
+    },
+    {
+      "epoch": 0.09890986439776656,
+      "grad_norm": 1.8731011152267456,
+      "learning_rate": 4.969201777632205e-05,
+      "loss": 1.6908,
+      "step": 2976
+    },
+    {
+      "epoch": 0.09994017548524328,
+      "grad_norm": 2.06569766998291,
+      "learning_rate": 4.9678736688350846e-05,
+      "loss": 1.6934,
+      "step": 3007
+    },
+    {
+      "epoch": 0.10097048657272002,
+      "grad_norm": 1.9683799743652344,
+      "learning_rate": 4.966517710419033e-05,
+      "loss": 1.6693,
+      "step": 3038
+    },
+    {
+      "epoch": 0.10200079766019676,
+      "grad_norm": 1.8517156839370728,
+      "learning_rate": 4.965133917685858e-05,
+      "loss": 1.704,
+      "step": 3069
+    },
+    {
+      "epoch": 0.10303110874767349,
+      "grad_norm": 1.9609614610671997,
+      "learning_rate": 4.9637223062514714e-05,
+      "loss": 1.6888,
+      "step": 3100
+    },
+    {
+      "epoch": 0.10406141983515023,
+      "grad_norm": 1.8825795650482178,
+      "learning_rate": 4.962282892045718e-05,
+      "loss": 1.6718,
+      "step": 3131
+    },
+    {
+      "epoch": 0.10509173092262696,
+      "grad_norm": 1.905633807182312,
+      "learning_rate": 4.9608156913121904e-05,
+      "loss": 1.715,
+      "step": 3162
+    },
+    {
+      "epoch": 0.1061220420101037,
+      "grad_norm": 1.8323174715042114,
+      "learning_rate": 4.959320720608049e-05,
+      "loss": 1.6817,
+      "step": 3193
+    },
+    {
+      "epoch": 0.10715235309758044,
+      "grad_norm": 1.9863115549087524,
+      "learning_rate": 4.9577979968038354e-05,
+      "loss": 1.6946,
+      "step": 3224
+    },
+    {
+      "epoch": 0.10818266418505716,
+      "grad_norm": 1.8490556478500366,
+      "learning_rate": 4.956247537083282e-05,
+      "loss": 1.6662,
+      "step": 3255
+    },
+    {
+      "epoch": 0.1092129752725339,
+      "grad_norm": 1.914162039756775,
+      "learning_rate": 4.9546693589431145e-05,
+      "loss": 1.6746,
+      "step": 3286
+    },
+    {
+      "epoch": 0.11024328636001064,
+      "grad_norm": 1.7593133449554443,
+      "learning_rate": 4.9530634801928595e-05,
+      "loss": 1.6803,
+      "step": 3317
+    },
+    {
+      "epoch": 0.11127359744748737,
+      "grad_norm": 1.7808279991149902,
+      "learning_rate": 4.9514299189546395e-05,
+      "loss": 1.6795,
+      "step": 3348
+    },
+    {
+      "epoch": 0.11230390853496411,
+      "grad_norm": 1.8970794677734375,
+      "learning_rate": 4.949768693662973e-05,
+      "loss": 1.6851,
+      "step": 3379
+    },
+    {
+      "epoch": 0.11333421962244083,
+      "grad_norm": 1.801310658454895,
+      "learning_rate": 4.948079823064559e-05,
+      "loss": 1.679,
+      "step": 3410
+    },
+    {
+      "epoch": 0.11436453070991758,
+      "grad_norm": 1.899228572845459,
+      "learning_rate": 4.946363326218074e-05,
+      "loss": 1.648,
+      "step": 3441
+    },
+    {
+      "epoch": 0.11539484179739432,
+      "grad_norm": 1.86049222946167,
+      "learning_rate": 4.9446192224939525e-05,
+      "loss": 1.6764,
+      "step": 3472
+    },
+    {
+      "epoch": 0.11642515288487104,
+      "grad_norm": 1.8451727628707886,
+      "learning_rate": 4.942847531574167e-05,
+      "loss": 1.6445,
+      "step": 3503
+    },
+    {
+      "epoch": 0.11745546397234778,
+      "grad_norm": 1.7101646661758423,
+      "learning_rate": 4.941048273452008e-05,
+      "loss": 1.668,
+      "step": 3534
+    },
+    {
+      "epoch": 0.11848577505982451,
+      "grad_norm": 1.8085941076278687,
+      "learning_rate": 4.9392214684318605e-05,
+      "loss": 1.6685,
+      "step": 3565
+    },
+    {
+      "epoch": 0.11951608614730125,
+      "grad_norm": 1.8603261709213257,
+      "learning_rate": 4.93736713712897e-05,
+      "loss": 1.6439,
+      "step": 3596
+    },
+    {
+      "epoch": 0.12054639723477799,
+      "grad_norm": 1.6646064519882202,
+      "learning_rate": 4.9354853004692124e-05,
+      "loss": 1.6498,
+      "step": 3627
+    },
+    {
+      "epoch": 0.12157670832225471,
+      "grad_norm": 1.724837064743042,
+      "learning_rate": 4.93357597968886e-05,
+      "loss": 1.6332,
+      "step": 3658
+    },
+    {
+      "epoch": 0.12260701940973145,
+      "grad_norm": 1.8698619604110718,
+      "learning_rate": 4.931639196334338e-05,
+      "loss": 1.6509,
+      "step": 3689
+    },
+    {
+      "epoch": 0.1236373304972082,
+      "grad_norm": 1.8970638513565063,
+      "learning_rate": 4.9296749722619826e-05,
+      "loss": 1.6641,
+      "step": 3720
+    },
+    {
+      "epoch": 0.12466764158468492,
+      "grad_norm": 1.812328577041626,
+      "learning_rate": 4.9276833296377966e-05,
+      "loss": 1.6387,
+      "step": 3751
+    },
+    {
+      "epoch": 0.12569795267216166,
+      "grad_norm": 1.746220350265503,
+      "learning_rate": 4.925664290937196e-05,
+      "loss": 1.6437,
+      "step": 3782
+    },
+    {
+      "epoch": 0.1267282637596384,
+      "grad_norm": 1.797878623008728,
+      "learning_rate": 4.9236178789447576e-05,
+      "loss": 1.639,
+      "step": 3813
+    },
+    {
+      "epoch": 0.12775857484711514,
+      "grad_norm": 1.8563851118087769,
+      "learning_rate": 4.921544116753962e-05,
+      "loss": 1.6531,
+      "step": 3844
+    },
+    {
+      "epoch": 0.12878888593459187,
+      "grad_norm": 1.7102128267288208,
+      "learning_rate": 4.919443027766935e-05,
+      "loss": 1.6354,
+      "step": 3875
+    },
+    {
+      "epoch": 0.1298191970220686,
+      "grad_norm": 1.924269676208496,
+      "learning_rate": 4.91731463569418e-05,
+      "loss": 1.6406,
+      "step": 3906
+    },
+    {
+      "epoch": 0.13084950810954532,
+      "grad_norm": 1.7960426807403564,
+      "learning_rate": 4.915158964554312e-05,
+      "loss": 1.6432,
+      "step": 3937
+    },
+    {
+      "epoch": 0.13187981919702207,
+      "grad_norm": 1.727726936340332,
+      "learning_rate": 4.912976038673786e-05,
+      "loss": 1.6378,
+      "step": 3968
+    },
+    {
+      "epoch": 0.1329101302844988,
+      "grad_norm": 1.7129536867141724,
+      "learning_rate": 4.9107658826866254e-05,
+      "loss": 1.6237,
+      "step": 3999
+    },
+    {
+      "epoch": 0.13394044137197553,
+      "grad_norm": 2.3489906787872314,
+      "learning_rate": 4.908528521534139e-05,
+      "loss": 1.6397,
+      "step": 4030
+    },
+    {
+      "epoch": 0.13497075245945228,
+      "grad_norm": 1.8773754835128784,
+      "learning_rate": 4.906263980464644e-05,
+      "loss": 1.6509,
+      "step": 4061
+    },
+    {
+      "epoch": 0.136001063546929,
+      "grad_norm": 1.7570220232009888,
+      "learning_rate": 4.903972285033178e-05,
+      "loss": 1.6094,
+      "step": 4092
+    },
+    {
+      "epoch": 0.13703137463440573,
+      "grad_norm": 1.8965568542480469,
+      "learning_rate": 4.901653461101213e-05,
+      "loss": 1.6253,
+      "step": 4123
+    },
+    {
+      "epoch": 0.1380616857218825,
+      "grad_norm": 1.7009938955307007,
+      "learning_rate": 4.8993075348363626e-05,
+      "loss": 1.6292,
+      "step": 4154
+    },
+    {
+      "epoch": 0.13909199680935921,
+      "grad_norm": 1.7146203517913818,
+      "learning_rate": 4.896934532712084e-05,
+      "loss": 1.6159,
+      "step": 4185
+    },
+    {
+      "epoch": 0.14012230789683594,
+      "grad_norm": 1.7641630172729492,
+      "learning_rate": 4.8945344815073846e-05,
+      "loss": 1.6285,
+      "step": 4216
+    },
+    {
+      "epoch": 0.1411526189843127,
+      "grad_norm": 1.5427709817886353,
+      "learning_rate": 4.892107408306516e-05,
+      "loss": 1.6305,
+      "step": 4247
+    },
+    {
+      "epoch": 0.14218293007178942,
+      "grad_norm": 1.72251558303833,
+      "learning_rate": 4.889653340498669e-05,
+      "loss": 1.6155,
+      "step": 4278
+    },
+    {
+      "epoch": 0.14321324115926615,
+      "grad_norm": 1.657377004623413,
+      "learning_rate": 4.8871723057776664e-05,
+      "loss": 1.6309,
+      "step": 4309
+    },
+    {
+      "epoch": 0.1442435522467429,
+      "grad_norm": 1.7477213144302368,
+      "learning_rate": 4.8846643321416476e-05,
+      "loss": 1.6252,
+      "step": 4340
+    },
+    {
+      "epoch": 0.14527386333421963,
+      "grad_norm": 1.7635339498519897,
+      "learning_rate": 4.882129447892753e-05,
+      "loss": 1.6371,
+      "step": 4371
+    },
+    {
+      "epoch": 0.14630417442169635,
+      "grad_norm": 1.7317392826080322,
+      "learning_rate": 4.8795676816368076e-05,
+      "loss": 1.6122,
+      "step": 4402
+    },
+    {
+      "epoch": 0.14733448550917308,
+      "grad_norm": 1.7233941555023193,
+      "learning_rate": 4.876979062282995e-05,
+      "loss": 1.6148,
+      "step": 4433
+    },
+    {
+      "epoch": 0.14836479659664983,
+      "grad_norm": 1.9179445505142212,
+      "learning_rate": 4.8743636190435325e-05,
+      "loss": 1.6157,
+      "step": 4464
+    },
+    {
+      "epoch": 0.14939510768412656,
+      "grad_norm": 1.722945213317871,
+      "learning_rate": 4.871721381433344e-05,
+      "loss": 1.626,
+      "step": 4495
+    },
+    {
+      "epoch": 0.1504254187716033,
+      "grad_norm": 1.6069978475570679,
+      "learning_rate": 4.869052379269719e-05,
+      "loss": 1.6137,
+      "step": 4526
+    },
+    {
+      "epoch": 0.15145572985908004,
+      "grad_norm": 1.6798020601272583,
+      "learning_rate": 4.866356642671985e-05,
+      "loss": 1.5982,
+      "step": 4557
+    },
+    {
+      "epoch": 0.15248604094655677,
+      "grad_norm": 1.678053855895996,
+      "learning_rate": 4.8636342020611634e-05,
+      "loss": 1.6139,
+      "step": 4588
+    },
+    {
+      "epoch": 0.1535163520340335,
+      "grad_norm": 1.7278486490249634,
+      "learning_rate": 4.860885088159626e-05,
+      "loss": 1.6096,
+      "step": 4619
+    },
+    {
+      "epoch": 0.15454666312151025,
+      "grad_norm": 1.6635969877243042,
+      "learning_rate": 4.858109331990751e-05,
+      "loss": 1.6093,
+      "step": 4650
+    },
+    {
+      "epoch": 0.15557697420898697,
+      "grad_norm": 1.6084243059158325,
+      "learning_rate": 4.855306964878567e-05,
+      "loss": 1.5883,
+      "step": 4681
+    },
+    {
+      "epoch": 0.1566072852964637,
+      "grad_norm": 1.7566964626312256,
+      "learning_rate": 4.8524780184474084e-05,
+      "loss": 1.6086,
+      "step": 4712
+    },
+    {
+      "epoch": 0.15763759638394045,
+      "grad_norm": 1.6619112491607666,
+      "learning_rate": 4.8496225246215496e-05,
+      "loss": 1.6285,
+      "step": 4743
+    },
+    {
+      "epoch": 0.15866790747141718,
+      "grad_norm": 1.6006278991699219,
+      "learning_rate": 4.8467405156248505e-05,
+      "loss": 1.6076,
+      "step": 4774
+    },
+    {
+      "epoch": 0.1596982185588939,
+      "grad_norm": 1.6931226253509521,
+      "learning_rate": 4.843832023980392e-05,
+      "loss": 1.6036,
+      "step": 4805
+    },
+    {
+      "epoch": 0.16072852964637063,
+      "grad_norm": 1.6883424520492554,
+      "learning_rate": 4.840897082510106e-05,
+      "loss": 1.5914,
+      "step": 4836
+    },
+    {
+      "epoch": 0.1617588407338474,
+      "grad_norm": 1.6018158197402954,
+      "learning_rate": 4.8379357243344084e-05,
+      "loss": 1.6149,
+      "step": 4867
+    },
+    {
+      "epoch": 0.1627891518213241,
+      "grad_norm": 1.5941047668457031,
+      "learning_rate": 4.8349479828718236e-05,
+      "loss": 1.6083,
+      "step": 4898
+    },
+    {
+      "epoch": 0.16381946290880084,
+      "grad_norm": 1.596794843673706,
+      "learning_rate": 4.8319338918386075e-05,
+      "loss": 1.5832,
+      "step": 4929
+    },
+    {
+      "epoch": 0.1648497739962776,
+      "grad_norm": 1.635367512702942,
+      "learning_rate": 4.828893485248369e-05,
+      "loss": 1.5983,
+      "step": 4960
+    },
+    {
+      "epoch": 0.16588008508375432,
+      "grad_norm": 1.5640513896942139,
+      "learning_rate": 4.825826797411682e-05,
+      "loss": 1.6023,
+      "step": 4991
+    },
+    {
+      "epoch": 0.16691039617123105,
+      "grad_norm": 1.7526532411575317,
+      "learning_rate": 4.822733862935702e-05,
+      "loss": 1.6015,
+      "step": 5022
+    },
+    {
+      "epoch": 0.1679407072587078,
+      "grad_norm": 1.739694595336914,
+      "learning_rate": 4.819614716723775e-05,
+      "loss": 1.6004,
+      "step": 5053
+    },
+    {
+      "epoch": 0.16897101834618453,
+      "grad_norm": 1.6435447931289673,
+      "learning_rate": 4.8164693939750425e-05,
+      "loss": 1.5909,
+      "step": 5084
+    },
+    {
+      "epoch": 0.17000132943366125,
+      "grad_norm": 1.7074060440063477,
+      "learning_rate": 4.813297930184042e-05,
+      "loss": 1.6131,
+      "step": 5115
+    },
+    {
+      "epoch": 0.171031640521138,
+      "grad_norm": 1.567400336265564,
+      "learning_rate": 4.810100361140314e-05,
+      "loss": 1.6054,
+      "step": 5146
+    },
+    {
+      "epoch": 0.17206195160861473,
+      "grad_norm": 1.6380565166473389,
+      "learning_rate": 4.8068767229279885e-05,
+      "loss": 1.5948,
+      "step": 5177
+    },
+    {
+      "epoch": 0.17309226269609146,
+      "grad_norm": 1.5382912158966064,
+      "learning_rate": 4.8036270519253854e-05,
+      "loss": 1.5893,
+      "step": 5208
+    },
+    {
+      "epoch": 0.1741225737835682,
+      "grad_norm": 1.59961998462677,
+      "learning_rate": 4.8003513848046e-05,
+      "loss": 1.575,
+      "step": 5239
+    },
+    {
+      "epoch": 0.17515288487104494,
+      "grad_norm": 1.8677598237991333,
+      "learning_rate": 4.79704975853109e-05,
+      "loss": 1.6078,
+      "step": 5270
+    },
+    {
+      "epoch": 0.17618319595852167,
+      "grad_norm": 1.6640410423278809,
+      "learning_rate": 4.793722210363262e-05,
+      "loss": 1.5923,
+      "step": 5301
+    },
+    {
+      "epoch": 0.1772135070459984,
+      "grad_norm": 1.515498161315918,
+      "learning_rate": 4.7903687778520414e-05,
+      "loss": 1.5971,
+      "step": 5332
+    },
+    {
+      "epoch": 0.17824381813347515,
+      "grad_norm": 1.6211729049682617,
+      "learning_rate": 4.7869894988404593e-05,
+      "loss": 1.599,
+      "step": 5363
+    },
+    {
+      "epoch": 0.17927412922095187,
+      "grad_norm": 1.5390729904174805,
+      "learning_rate": 4.783584411463221e-05,
+      "loss": 1.5958,
+      "step": 5394
+    },
+    {
+      "epoch": 0.1803044403084286,
+      "grad_norm": 1.588283658027649,
+      "learning_rate": 4.780153554146274e-05,
+      "loss": 1.588,
+      "step": 5425
+    },
+    {
+      "epoch": 0.18133475139590535,
+      "grad_norm": 1.5678156614303589,
+      "learning_rate": 4.7766969656063766e-05,
+      "loss": 1.5839,
+      "step": 5456
+    },
+    {
+      "epoch": 0.18236506248338208,
+      "grad_norm": 1.6620889902114868,
+      "learning_rate": 4.773214684850662e-05,
+      "loss": 1.5993,
+      "step": 5487
+    },
+    {
+      "epoch": 0.1833953735708588,
+      "grad_norm": 1.5112054347991943,
+      "learning_rate": 4.769706751176193e-05,
+      "loss": 1.5765,
+      "step": 5518
+    },
+    {
+      "epoch": 0.18442568465833556,
+      "grad_norm": 1.5224021673202515,
+      "learning_rate": 4.7661732041695264e-05,
+      "loss": 1.6023,
+      "step": 5549
+    },
+    {
+      "epoch": 0.18545599574581229,
+      "grad_norm": 1.5449796915054321,
+      "learning_rate": 4.762614083706258e-05,
+      "loss": 1.5906,
+      "step": 5580
+    },
+    {
+      "epoch": 0.186486306833289,
+      "grad_norm": 1.571778416633606,
+      "learning_rate": 4.759029429950581e-05,
+      "loss": 1.5972,
+      "step": 5611
+    },
+    {
+      "epoch": 0.18751661792076577,
+      "grad_norm": 1.5014352798461914,
+      "learning_rate": 4.7554192833548235e-05,
+      "loss": 1.576,
+      "step": 5642
+    },
+    {
+      "epoch": 0.1885469290082425,
+      "grad_norm": 1.5657117366790771,
+      "learning_rate": 4.751783684659e-05,
+      "loss": 1.5797,
+      "step": 5673
+    },
+    {
+      "epoch": 0.18957724009571922,
+      "grad_norm": 1.523395299911499,
+      "learning_rate": 4.748122674890348e-05,
+      "loss": 1.5879,
+      "step": 5704
+    },
+    {
+      "epoch": 0.19060755118319597,
+      "grad_norm": 1.5439683198928833,
+      "learning_rate": 4.7444362953628654e-05,
+      "loss": 1.5653,
+      "step": 5735
+    },
+    {
+      "epoch": 0.1916378622706727,
+      "grad_norm": 1.6559383869171143,
+      "learning_rate": 4.7407245876768424e-05,
+      "loss": 1.5779,
+      "step": 5766
+    },
+    {
+      "epoch": 0.19266817335814942,
+      "grad_norm": 1.6374629735946655,
+      "learning_rate": 4.736987593718397e-05,
+      "loss": 1.5597,
+      "step": 5797
+    },
+    {
+      "epoch": 0.19369848444562615,
+      "grad_norm": 1.6622729301452637,
+      "learning_rate": 4.733225355658999e-05,
+      "loss": 1.5706,
+      "step": 5828
+    },
+    {
+      "epoch": 0.1947287955331029,
+      "grad_norm": 1.5311403274536133,
+      "learning_rate": 4.7294379159549926e-05,
+      "loss": 1.5719,
+      "step": 5859
+    },
+    {
+      "epoch": 0.19575910662057963,
+      "grad_norm": 1.5591763257980347,
+      "learning_rate": 4.725625317347119e-05,
+      "loss": 1.5945,
+      "step": 5890
+    },
+    {
+      "epoch": 0.19678941770805636,
+      "grad_norm": 1.6840064525604248,
+      "learning_rate": 4.7217876028600374e-05,
+      "loss": 1.5661,
+      "step": 5921
+    },
+    {
+      "epoch": 0.1978197287955331,
+      "grad_norm": 1.5817420482635498,
+      "learning_rate": 4.717924815801832e-05,
+      "loss": 1.5627,
+      "step": 5952
+    },
+    {
+      "epoch": 0.19885003988300984,
+      "grad_norm": 1.4626916646957397,
+      "learning_rate": 4.714036999763532e-05,
+      "loss": 1.5668,
+      "step": 5983
+    },
+    {
+      "epoch": 0.19988035097048656,
+      "grad_norm": 1.6004570722579956,
+      "learning_rate": 4.7101241986186116e-05,
+      "loss": 1.58,
+      "step": 6014
+    },
+    {
+      "epoch": 0.20091066205796332,
+      "grad_norm": 1.565014123916626,
+      "learning_rate": 4.7061864565225e-05,
+      "loss": 1.5642,
+      "step": 6045
+    },
+    {
+      "epoch": 0.20194097314544004,
+      "grad_norm": 1.523751139640808,
+      "learning_rate": 4.702223817912081e-05,
+      "loss": 1.575,
+      "step": 6076
+    },
+    {
+      "epoch": 0.20297128423291677,
+      "grad_norm": 1.531226396560669,
+      "learning_rate": 4.698236327505195e-05,
+      "loss": 1.5531,
+      "step": 6107
+    },
+    {
+      "epoch": 0.20400159532039353,
+      "grad_norm": 1.569403052330017,
+      "learning_rate": 4.694224030300127e-05,
+      "loss": 1.5655,
+      "step": 6138
+    },
+    {
+      "epoch": 0.20503190640787025,
+      "grad_norm": 1.607932209968567,
+      "learning_rate": 4.690186971575107e-05,
+      "loss": 1.5568,
+      "step": 6169
+    },
+    {
+      "epoch": 0.20606221749534698,
+      "grad_norm": 1.4806853532791138,
+      "learning_rate": 4.6861251968877916e-05,
+      "loss": 1.568,
+      "step": 6200
+    },
+    {
+      "epoch": 0.2070925285828237,
+      "grad_norm": 1.606819748878479,
+      "learning_rate": 4.68203875207476e-05,
+      "loss": 1.5711,
+      "step": 6231
+    },
+    {
+      "epoch": 0.20812283967030046,
+      "grad_norm": 1.4872000217437744,
+      "learning_rate": 4.677927683250983e-05,
+      "loss": 1.5583,
+      "step": 6262
+    },
+    {
+      "epoch": 0.20915315075777718,
+      "grad_norm": 1.5292075872421265,
+      "learning_rate": 4.6737920368093156e-05,
+      "loss": 1.5528,
+      "step": 6293
+    },
+    {
+      "epoch": 0.2101834618452539,
+      "grad_norm": 1.4633690118789673,
+      "learning_rate": 4.669631859419965e-05,
+      "loss": 1.5526,
+      "step": 6324
+    },
+    {
+      "epoch": 0.21121377293273066,
+      "grad_norm": 1.4584565162658691,
+      "learning_rate": 4.6654471980299676e-05,
+      "loss": 1.5632,
+      "step": 6355
+    },
+    {
+      "epoch": 0.2122440840202074,
+      "grad_norm": 1.4834545850753784,
+      "learning_rate": 4.661238099862658e-05,
+      "loss": 1.5722,
+      "step": 6386
+    },
+    {
+      "epoch": 0.21327439510768412,
+      "grad_norm": 1.5098270177841187,
+      "learning_rate": 4.657004612417138e-05,
+      "loss": 1.5676,
+      "step": 6417
+    },
+    {
+      "epoch": 0.21430470619516087,
+      "grad_norm": 1.4672640562057495,
+      "learning_rate": 4.6527467834677374e-05,
+      "loss": 1.5534,
+      "step": 6448
+    },
+    {
+      "epoch": 0.2153350172826376,
+      "grad_norm": 1.451723337173462,
+      "learning_rate": 4.648464661063478e-05,
+      "loss": 1.5652,
+      "step": 6479
+    },
+    {
+      "epoch": 0.21636532837011432,
+      "grad_norm": 1.509397029876709,
+      "learning_rate": 4.6441582935275264e-05,
+      "loss": 1.5585,
+      "step": 6510
+    },
+    {
+      "epoch": 0.21739563945759108,
+      "grad_norm": 1.4893388748168945,
+      "learning_rate": 4.6398277294566586e-05,
+      "loss": 1.5524,
+      "step": 6541
+    },
+    {
+      "epoch": 0.2184259505450678,
+      "grad_norm": 1.48307466506958,
+      "learning_rate": 4.6354730177207e-05,
+      "loss": 1.5446,
+      "step": 6572
+    },
+    {
+      "epoch": 0.21945626163254453,
+      "grad_norm": 1.6026273965835571,
+      "learning_rate": 4.6310942074619787e-05,
+      "loss": 1.5742,
+      "step": 6603
+    },
+    {
+      "epoch": 0.22048657272002128,
+      "grad_norm": 1.4466279745101929,
+      "learning_rate": 4.626691348094777e-05,
+      "loss": 1.5338,
+      "step": 6634
+    },
+    {
+      "epoch": 0.221516883807498,
+      "grad_norm": 1.4488214254379272,
+      "learning_rate": 4.622264489304762e-05,
+      "loss": 1.5486,
+      "step": 6665
+    },
+    {
+      "epoch": 0.22254719489497474,
+      "grad_norm": 1.523085355758667,
+      "learning_rate": 4.617813681048434e-05,
+      "loss": 1.5793,
+      "step": 6696
+    },
+    {
+      "epoch": 0.22357750598245146,
+      "grad_norm": 1.513340711593628,
+      "learning_rate": 4.61333897355256e-05,
+      "loss": 1.5465,
+      "step": 6727
+    },
+    {
+      "epoch": 0.22460781706992822,
+      "grad_norm": 1.5583122968673706,
+      "learning_rate": 4.608840417313604e-05,
+      "loss": 1.5707,
+      "step": 6758
+    },
+    {
+      "epoch": 0.22563812815740494,
+      "grad_norm": 1.4766571521759033,
+      "learning_rate": 4.6043180630971646e-05,
+      "loss": 1.5701,
+      "step": 6789
+    },
+    {
+      "epoch": 0.22666843924488167,
+      "grad_norm": 1.5157111883163452,
+      "learning_rate": 4.599771961937391e-05,
+      "loss": 1.5547,
+      "step": 6820
+    },
+    {
+      "epoch": 0.22769875033235842,
+      "grad_norm": 1.558409571647644,
+      "learning_rate": 4.5952021651364204e-05,
+      "loss": 1.5351,
+      "step": 6851
+    },
+    {
+      "epoch": 0.22872906141983515,
+      "grad_norm": 1.4935834407806396,
+      "learning_rate": 4.590608724263786e-05,
+      "loss": 1.5697,
+      "step": 6882
+    },
+    {
+      "epoch": 0.22975937250731188,
+      "grad_norm": 1.4702025651931763,
+      "learning_rate": 4.585991691155845e-05,
+      "loss": 1.5733,
+      "step": 6913
+    },
+    {
+      "epoch": 0.23078968359478863,
+      "grad_norm": 1.4582765102386475,
+      "learning_rate": 4.581351117915188e-05,
+      "loss": 1.541,
+      "step": 6944
+    },
+    {
+      "epoch": 0.23181999468226536,
+      "grad_norm": 1.4955871105194092,
+      "learning_rate": 4.5766870569100534e-05,
+      "loss": 1.5436,
+      "step": 6975
+    },
+    {
+      "epoch": 0.23285030576974208,
+      "grad_norm": 1.5541787147521973,
+      "learning_rate": 4.571999560773736e-05,
+      "loss": 1.5531,
+      "step": 7006
+    },
+    {
+      "epoch": 0.23388061685721884,
+      "grad_norm": 1.4085042476654053,
+      "learning_rate": 4.5672886824039915e-05,
+      "loss": 1.5474,
+      "step": 7037
+    },
+    {
+      "epoch": 0.23491092794469556,
+      "grad_norm": 1.4347487688064575,
+      "learning_rate": 4.5625544749624435e-05,
+      "loss": 1.5562,
+      "step": 7068
+    },
+    {
+      "epoch": 0.2359412390321723,
+      "grad_norm": 1.4909547567367554,
+      "learning_rate": 4.5577969918739794e-05,
+      "loss": 1.5545,
+      "step": 7099
+    },
+    {
+      "epoch": 0.23697155011964902,
+      "grad_norm": 1.5197391510009766,
+      "learning_rate": 4.5530162868261486e-05,
+      "loss": 1.5447,
+      "step": 7130
+    },
+    {
+      "epoch": 0.23800186120712577,
+      "grad_norm": 1.4785538911819458,
+      "learning_rate": 4.548212413768558e-05,
+      "loss": 1.5436,
+      "step": 7161
+    },
+    {
+      "epoch": 0.2390321722946025,
+      "grad_norm": 1.4719116687774658,
+      "learning_rate": 4.543385426912261e-05,
+      "loss": 1.5404,
+      "step": 7192
+    },
+    {
+      "epoch": 0.24006248338207922,
+      "grad_norm": 1.5343807935714722,
+      "learning_rate": 4.53853538072915e-05,
+      "loss": 1.5567,
+      "step": 7223
+    },
+    {
+      "epoch": 0.24109279446955598,
+      "grad_norm": 1.4943312406539917,
+      "learning_rate": 4.533662329951336e-05,
+      "loss": 1.5643,
+      "step": 7254
+    },
+    {
+      "epoch": 0.2421231055570327,
+      "grad_norm": 1.5655717849731445,
+      "learning_rate": 4.528766329570536e-05,
+      "loss": 1.5413,
+      "step": 7285
+    },
+    {
+      "epoch": 0.24315341664450943,
+      "grad_norm": 2.006964683532715,
+      "learning_rate": 4.523847434837447e-05,
+      "loss": 1.546,
+      "step": 7316
+    },
+    {
+      "epoch": 0.24418372773198618,
+      "grad_norm": 1.3473403453826904,
+      "learning_rate": 4.518905701261128e-05,
+      "loss": 1.5438,
+      "step": 7347
+    },
+    {
+      "epoch": 0.2452140388194629,
+      "grad_norm": 1.4880343675613403,
+      "learning_rate": 4.5139411846083715e-05,
+      "loss": 1.5467,
+      "step": 7378
+    },
+    {
+      "epoch": 0.24624434990693964,
+      "grad_norm": 1.4092179536819458,
+      "learning_rate": 4.508953940903073e-05,
+      "loss": 1.5391,
+      "step": 7409
+    },
+    {
+      "epoch": 0.2472746609944164,
+      "grad_norm": 1.5732282400131226,
+      "learning_rate": 4.5039440264255994e-05,
+      "loss": 1.5498,
+      "step": 7440
+    },
+    {
+      "epoch": 0.24830497208189312,
+      "grad_norm": 3.3359272480010986,
+      "learning_rate": 4.498911497712155e-05,
+      "loss": 1.5446,
+      "step": 7471
+    },
+    {
+      "epoch": 0.24933528316936984,
+      "grad_norm": 1.4734565019607544,
+      "learning_rate": 4.493856411554142e-05,
+      "loss": 1.5468,
+      "step": 7502
+    },
+    {
+      "epoch": 0.25036559425684657,
+      "grad_norm": 1.5382531881332397,
+      "learning_rate": 4.4887788249975206e-05,
+      "loss": 1.5431,
+      "step": 7533
+    },
+    {
+      "epoch": 0.2513959053443233,
+      "grad_norm": 1.5253698825836182,
+      "learning_rate": 4.4836787953421656e-05,
+      "loss": 1.5373,
+      "step": 7564
+    },
+    {
+      "epoch": 0.2524262164318001,
+      "grad_norm": 1.5616925954818726,
+      "learning_rate": 4.478556380141218e-05,
+      "loss": 1.5695,
+      "step": 7595
+    },
+    {
+      "epoch": 0.2534565275192768,
+      "grad_norm": 1.4779139757156372,
+      "learning_rate": 4.4734116372004375e-05,
+      "loss": 1.5423,
+      "step": 7626
+    },
+    {
+      "epoch": 0.25448683860675353,
+      "grad_norm": 1.4286718368530273,
+      "learning_rate": 4.4682446245775477e-05,
+      "loss": 1.5434,
+      "step": 7657
+    },
+    {
+      "epoch": 0.2555171496942303,
+      "grad_norm": 1.559987187385559,
+      "learning_rate": 4.463055400581586e-05,
+      "loss": 1.5402,
+      "step": 7688
+    },
+    {
+      "epoch": 0.256547460781707,
+      "grad_norm": 1.4642205238342285,
+      "learning_rate": 4.4578440237722374e-05,
+      "loss": 1.5409,
+      "step": 7719
+    },
+    {
+      "epoch": 0.25757777186918374,
+      "grad_norm": 1.447375774383545,
+      "learning_rate": 4.452610552959183e-05,
+      "loss": 1.5353,
+      "step": 7750
+    },
+    {
+      "epoch": 0.2586080829566605,
+      "grad_norm": 1.4967209100723267,
+      "learning_rate": 4.447355047201428e-05,
+      "loss": 1.5432,
+      "step": 7781
+    },
+    {
+      "epoch": 0.2596383940441372,
+      "grad_norm": 1.4597041606903076,
+      "learning_rate": 4.4420775658066414e-05,
+      "loss": 1.5355,
+      "step": 7812
+    },
+    {
+      "epoch": 0.26066870513161394,
+      "grad_norm": 1.3647087812423706,
+      "learning_rate": 4.436778168330484e-05,
+      "loss": 1.5413,
+      "step": 7843
+    },
+    {
+      "epoch": 0.26169901621909064,
+      "grad_norm": 1.3664494752883911,
+      "learning_rate": 4.4314569145759353e-05,
+      "loss": 1.5194,
+      "step": 7874
+    },
+    {
+      "epoch": 0.2627293273065674,
+      "grad_norm": 1.4098619222640991,
+      "learning_rate": 4.42611386459262e-05,
+      "loss": 1.5369,
+      "step": 7905
+    },
+    {
+      "epoch": 0.26375963839404415,
+      "grad_norm": 1.468239665031433,
+      "learning_rate": 4.420749078676133e-05,
+      "loss": 1.5093,
+      "step": 7936
+    },
+    {
+      "epoch": 0.26478994948152085,
+      "grad_norm": 1.5132948160171509,
+      "learning_rate": 4.4153626173673516e-05,
+      "loss": 1.5258,
+      "step": 7967
+    },
+    {
+      "epoch": 0.2658202605689976,
+      "grad_norm": 1.4278615713119507,
+      "learning_rate": 4.409954541451762e-05,
+      "loss": 1.5515,
+      "step": 7998
+    },
+    {
+      "epoch": 0.26685057165647436,
+      "grad_norm": 1.5070613622665405,
+      "learning_rate": 4.404524911958764e-05,
+      "loss": 1.5295,
+      "step": 8029
+    },
+    {
+      "epoch": 0.26788088274395105,
+      "grad_norm": 1.4723968505859375,
+      "learning_rate": 4.399073790160989e-05,
+      "loss": 1.5453,
+      "step": 8060
+    },
+    {
+      "epoch": 0.2689111938314278,
+      "grad_norm": 1.399830937385559,
+      "learning_rate": 4.393601237573607e-05,
+      "loss": 1.5412,
+      "step": 8091
+    },
+    {
+      "epoch": 0.26994150491890456,
+      "grad_norm": 1.4758100509643555,
+      "learning_rate": 4.388107315953628e-05,
+      "loss": 1.5454,
+      "step": 8122
+    },
+    {
+      "epoch": 0.27097181600638126,
+      "grad_norm": 1.423598051071167,
+      "learning_rate": 4.382592087299212e-05,
+      "loss": 1.5361,
+      "step": 8153
+    },
+    {
+      "epoch": 0.272002127093858,
+      "grad_norm": 1.4820687770843506,
+      "learning_rate": 4.377055613848964e-05,
+      "loss": 1.5037,
+      "step": 8184
+    },
+    {
+      "epoch": 0.27303243818133477,
+      "grad_norm": 1.4939546585083008,
+      "learning_rate": 4.3714979580812355e-05,
+      "loss": 1.5168,
+      "step": 8215
+    },
+    {
+      "epoch": 0.27406274926881147,
+      "grad_norm": 1.4235663414001465,
+      "learning_rate": 4.365919182713416e-05,
+      "loss": 1.5215,
+      "step": 8246
+    },
+    {
+      "epoch": 0.2750930603562882,
+      "grad_norm": 1.4634205102920532,
+      "learning_rate": 4.360319350701226e-05,
+      "loss": 1.5218,
+      "step": 8277
+    },
+    {
+      "epoch": 0.276123371443765,
+      "grad_norm": 1.4316506385803223,
+      "learning_rate": 4.3546985252380115e-05,
+      "loss": 1.5301,
+      "step": 8308
+    },
+    {
+      "epoch": 0.2771536825312417,
+      "grad_norm": 1.416364073753357,
+      "learning_rate": 4.349056769754021e-05,
+      "loss": 1.524,
+      "step": 8339
+    },
+    {
+      "epoch": 0.27818399361871843,
+      "grad_norm": 1.4837143421173096,
+      "learning_rate": 4.3433941479156994e-05,
+      "loss": 1.5372,
+      "step": 8370
+    },
+    {
+      "epoch": 0.2792143047061952,
+      "grad_norm": 1.4120386838912964,
+      "learning_rate": 4.3377107236249647e-05,
+      "loss": 1.5085,
+      "step": 8401
+    },
+    {
+      "epoch": 0.2802446157936719,
+      "grad_norm": 1.4150009155273438,
+      "learning_rate": 4.332006561018488e-05,
+      "loss": 1.5188,
+      "step": 8432
+    },
+    {
+      "epoch": 0.28127492688114863,
+      "grad_norm": 1.3972294330596924,
+      "learning_rate": 4.3262817244669683e-05,
+      "loss": 1.5167,
+      "step": 8463
+    },
+    {
+      "epoch": 0.2823052379686254,
+      "grad_norm": 1.4402275085449219,
+      "learning_rate": 4.3205362785744083e-05,
+      "loss": 1.5365,
+      "step": 8494
+    },
+    {
+      "epoch": 0.2833355490561021,
+      "grad_norm": 1.5087653398513794,
+      "learning_rate": 4.314770288177384e-05,
+      "loss": 1.5252,
+      "step": 8525
+    },
+    {
+      "epoch": 0.28436586014357884,
+      "grad_norm": 1.4362132549285889,
+      "learning_rate": 4.308983818344313e-05,
+      "loss": 1.5302,
+      "step": 8556
+    },
+    {
+      "epoch": 0.2853961712310556,
+      "grad_norm": 1.3816102743148804,
+      "learning_rate": 4.3031769343747206e-05,
+      "loss": 1.5264,
+      "step": 8587
+    },
+    {
+      "epoch": 0.2864264823185323,
+      "grad_norm": 1.5195884704589844,
+      "learning_rate": 4.297349701798505e-05,
+      "loss": 1.5051,
+      "step": 8618
+    },
+    {
+      "epoch": 0.28745679340600905,
+      "grad_norm": 1.7179895639419556,
+      "learning_rate": 4.2915021863751916e-05,
+      "loss": 1.5224,
+      "step": 8649
+    },
+    {
+      "epoch": 0.2884871044934858,
+      "grad_norm": 1.526225209236145,
+      "learning_rate": 4.285634454093198e-05,
+      "loss": 1.5035,
+      "step": 8680
+    },
+    {
+      "epoch": 0.2895174155809625,
+      "grad_norm": 1.425753116607666,
+      "learning_rate": 4.279746571169086e-05,
+      "loss": 1.5006,
+      "step": 8711
+    },
+    {
+      "epoch": 0.29054772666843925,
+      "grad_norm": 1.3119168281555176,
+      "learning_rate": 4.2738386040468136e-05,
+      "loss": 1.5196,
+      "step": 8742
+    },
+    {
+      "epoch": 0.29157803775591595,
+      "grad_norm": 1.462497591972351,
+      "learning_rate": 4.2679106193969866e-05,
+      "loss": 1.4969,
+      "step": 8773
+    },
+    {
+      "epoch": 0.2926083488433927,
+      "grad_norm": 1.4374085664749146,
+      "learning_rate": 4.261962684116106e-05,
+      "loss": 1.5152,
+      "step": 8804
+    },
+    {
+      "epoch": 0.29363865993086946,
+      "grad_norm": 1.3366769552230835,
+      "learning_rate": 4.2559948653258145e-05,
+      "loss": 1.507,
+      "step": 8835
+    },
+    {
+      "epoch": 0.29466897101834616,
+      "grad_norm": 1.4314748048782349,
+      "learning_rate": 4.250007230372134e-05,
+      "loss": 1.5301,
+      "step": 8866
+    },
+    {
+      "epoch": 0.2956992821058229,
+      "grad_norm": 1.375900387763977,
+      "learning_rate": 4.2439998468247126e-05,
+      "loss": 1.5137,
+      "step": 8897
+    },
+    {
+      "epoch": 0.29672959319329967,
+      "grad_norm": 1.3341666460037231,
+      "learning_rate": 4.2379727824760566e-05,
+      "loss": 1.522,
+      "step": 8928
+    },
+    {
+      "epoch": 0.29775990428077637,
+      "grad_norm": 1.345363736152649,
+      "learning_rate": 4.231926105340768e-05,
+      "loss": 1.4951,
+      "step": 8959
+    },
+    {
+      "epoch": 0.2987902153682531,
+      "grad_norm": 1.4105278253555298,
+      "learning_rate": 4.225859883654776e-05,
+      "loss": 1.5272,
+      "step": 8990
+    },
+    {
+      "epoch": 0.2998205264557299,
+      "grad_norm": 1.3522354364395142,
+      "learning_rate": 4.219774185874569e-05,
+      "loss": 1.5319,
+      "step": 9021
+    },
+    {
+      "epoch": 0.3008508375432066,
+      "grad_norm": 1.3407599925994873,
+      "learning_rate": 4.213669080676418e-05,
+      "loss": 1.5346,
+      "step": 9052
+    },
+    {
+      "epoch": 0.3018811486306833,
+      "grad_norm": 1.4593051671981812,
+      "learning_rate": 4.2075446369556056e-05,
+      "loss": 1.5122,
+      "step": 9083
+    },
+    {
+      "epoch": 0.3029114597181601,
+      "grad_norm": 1.3011035919189453,
+      "learning_rate": 4.201400923825648e-05,
+      "loss": 1.5076,
+      "step": 9114
+    },
+    {
+      "epoch": 0.3039417708056368,
+      "grad_norm": 1.431721806526184,
+      "learning_rate": 4.195238010617511e-05,
+      "loss": 1.5152,
+      "step": 9145
+    },
+    {
+      "epoch": 0.30497208189311353,
+      "grad_norm": 1.4024991989135742,
+      "learning_rate": 4.1890559668788344e-05,
+      "loss": 1.5101,
+      "step": 9176
+    },
+    {
+      "epoch": 0.3060023929805903,
+      "grad_norm": 1.3289436101913452,
+      "learning_rate": 4.1828548623731405e-05,
+      "loss": 1.4972,
+      "step": 9207
+    },
+    {
+      "epoch": 0.307032704068067,
+      "grad_norm": 1.4914730787277222,
+      "learning_rate": 4.1766347670790506e-05,
+      "loss": 1.5038,
+      "step": 9238
+    },
+    {
+      "epoch": 0.30806301515554374,
+      "grad_norm": 1.3376572132110596,
+      "learning_rate": 4.170395751189495e-05,
+      "loss": 1.5202,
+      "step": 9269
+    },
+    {
+      "epoch": 0.3090933262430205,
+      "grad_norm": 1.3964329957962036,
+      "learning_rate": 4.164137885110921e-05,
+      "loss": 1.4875,
+      "step": 9300
+    },
+    {
+      "epoch": 0.3101236373304972,
+      "grad_norm": 1.2990026473999023,
+      "learning_rate": 4.157861239462495e-05,
+      "loss": 1.5051,
+      "step": 9331
+    },
+    {
+      "epoch": 0.31115394841797395,
+      "grad_norm": 1.3120143413543701,
+      "learning_rate": 4.1515658850753114e-05,
+      "loss": 1.4965,
+      "step": 9362
+    },
+    {
+      "epoch": 0.3121842595054507,
+      "grad_norm": 1.3370813131332397,
+      "learning_rate": 4.145251892991588e-05,
+      "loss": 1.5081,
+      "step": 9393
+    },
+    {
+      "epoch": 0.3132145705929274,
+      "grad_norm": 1.376318097114563,
+      "learning_rate": 4.138919334463868e-05,
+      "loss": 1.4931,
+      "step": 9424
+    },
+    {
+      "epoch": 0.31424488168040415,
+      "grad_norm": 1.3892277479171753,
+      "learning_rate": 4.1325682809542124e-05,
+      "loss": 1.4998,
+      "step": 9455
+    },
+    {
+      "epoch": 0.3152751927678809,
+      "grad_norm": 1.31952965259552,
+      "learning_rate": 4.126198804133398e-05,
+      "loss": 1.5228,
+      "step": 9486
+    },
+    {
+      "epoch": 0.3163055038553576,
+      "grad_norm": 1.3445535898208618,
+      "learning_rate": 4.1198109758801055e-05,
+      "loss": 1.5254,
+      "step": 9517
+    },
+    {
+      "epoch": 0.31733581494283436,
+      "grad_norm": 1.3134032487869263,
+      "learning_rate": 4.113404868280107e-05,
+      "loss": 1.487,
+      "step": 9548
+    },
+    {
+      "epoch": 0.3183661260303111,
+      "grad_norm": 1.3956280946731567,
+      "learning_rate": 4.106980553625457e-05,
+      "loss": 1.5189,
+      "step": 9579
+    },
+    {
+      "epoch": 0.3193964371177878,
+      "grad_norm": 1.415956974029541,
+      "learning_rate": 4.100538104413674e-05,
+      "loss": 1.4853,
+      "step": 9610
+    },
+    {
+      "epoch": 0.32042674820526457,
+      "grad_norm": 1.333051323890686,
+      "learning_rate": 4.09407759334692e-05,
+      "loss": 1.4887,
+      "step": 9641
+    },
+    {
+      "epoch": 0.32145705929274127,
+      "grad_norm": 1.4168857336044312,
+      "learning_rate": 4.087599093331186e-05,
+      "loss": 1.4906,
+      "step": 9672
+    },
+    {
+      "epoch": 0.322487370380218,
+      "grad_norm": 1.452917218208313,
+      "learning_rate": 4.081102677475462e-05,
+      "loss": 1.5142,
+      "step": 9703
+    },
+    {
+      "epoch": 0.3235176814676948,
+      "grad_norm": 1.3834184408187866,
+      "learning_rate": 4.0745884190909194e-05,
+      "loss": 1.4924,
+      "step": 9734
+    },
+    {
+      "epoch": 0.32454799255517147,
+      "grad_norm": 1.3319405317306519,
+      "learning_rate": 4.0680563916900796e-05,
+      "loss": 1.5088,
+      "step": 9765
+    },
+    {
+      "epoch": 0.3255783036426482,
+      "grad_norm": 1.392349362373352,
+      "learning_rate": 4.0615066689859815e-05,
+      "loss": 1.5257,
+      "step": 9796
+    },
+    {
+      "epoch": 0.326608614730125,
+      "grad_norm": 1.3074041604995728,
+      "learning_rate": 4.0549393248913584e-05,
+      "loss": 1.501,
+      "step": 9827
+    },
+    {
+      "epoch": 0.3276389258176017,
+      "grad_norm": 1.3147920370101929,
+      "learning_rate": 4.048354433517794e-05,
+      "loss": 1.4924,
+      "step": 9858
+    },
+    {
+      "epoch": 0.32866923690507843,
+      "grad_norm": 1.3868871927261353,
+      "learning_rate": 4.0417520691748916e-05,
+      "loss": 1.507,
+      "step": 9889
+    },
+    {
+      "epoch": 0.3296995479925552,
+      "grad_norm": 1.3428276777267456,
+      "learning_rate": 4.035132306369438e-05,
+      "loss": 1.4975,
+      "step": 9920
+    },
+    {
+      "epoch": 0.3307298590800319,
+      "grad_norm": 1.3915890455245972,
+      "learning_rate": 4.028495219804555e-05,
+      "loss": 1.5037,
+      "step": 9951
+    },
+    {
+      "epoch": 0.33176017016750864,
+      "grad_norm": 1.3039841651916504,
+      "learning_rate": 4.021840884378864e-05,
+      "loss": 1.5157,
+      "step": 9982
+    },
+    {
+      "epoch": 0.3327904812549854,
+      "grad_norm": 1.370236873626709,
+      "learning_rate": 4.015169375185633e-05,
+      "loss": 1.4959,
+      "step": 10013
+    },
+    {
+      "epoch": 0.3338207923424621,
+      "grad_norm": 1.3154406547546387,
+      "learning_rate": 4.0084807675119396e-05,
+      "loss": 1.5017,
+      "step": 10044
+    },
+    {
+      "epoch": 0.33485110342993885,
+      "grad_norm": 1.4002853631973267,
+      "learning_rate": 4.0017751368378106e-05,
+      "loss": 1.4935,
+      "step": 10075
+    },
+    {
+      "epoch": 0.3358814145174156,
+      "grad_norm": 1.327405571937561,
+      "learning_rate": 3.995052558835377e-05,
+      "loss": 1.4927,
+      "step": 10106
+    },
+    {
+      "epoch": 0.3369117256048923,
+      "grad_norm": 1.3635532855987549,
+      "learning_rate": 3.988313109368017e-05,
+      "loss": 1.5051,
+      "step": 10137
+    },
+    {
+      "epoch": 0.33794203669236905,
+      "grad_norm": 1.3702609539031982,
+      "learning_rate": 3.981556864489504e-05,
+      "loss": 1.5114,
+      "step": 10168
+    },
+    {
+      "epoch": 0.3389723477798458,
+      "grad_norm": 1.4702330827713013,
+      "learning_rate": 3.974783900443142e-05,
+      "loss": 1.4998,
+      "step": 10199
+    },
+    {
+      "epoch": 0.3400026588673225,
+      "grad_norm": 1.3873344659805298,
+      "learning_rate": 3.9679942936609095e-05,
+      "loss": 1.5023,
+      "step": 10230
+    },
+    {
+      "epoch": 0.34103296995479926,
+      "grad_norm": 1.3811312913894653,
+      "learning_rate": 3.961188120762596e-05,
+      "loss": 1.4855,
+      "step": 10261
+    },
+    {
+      "epoch": 0.342063281042276,
+      "grad_norm": 1.2822514772415161,
+      "learning_rate": 3.954365458554938e-05,
+      "loss": 1.4987,
+      "step": 10292
+    },
+    {
+      "epoch": 0.3430935921297527,
+      "grad_norm": 1.3461897373199463,
+      "learning_rate": 3.947526384030751e-05,
+      "loss": 1.4994,
+      "step": 10323
+    },
+    {
+      "epoch": 0.34412390321722947,
+      "grad_norm": 1.3947008848190308,
+      "learning_rate": 3.9406709743680624e-05,
+      "loss": 1.4862,
+      "step": 10354
+    },
+    {
+      "epoch": 0.3451542143047062,
+      "grad_norm": 1.3745840787887573,
+      "learning_rate": 3.9337993069292366e-05,
+      "loss": 1.4869,
+      "step": 10385
+    },
+    {
+      "epoch": 0.3461845253921829,
+      "grad_norm": 1.3414299488067627,
+      "learning_rate": 3.926911459260109e-05,
+      "loss": 1.4774,
+      "step": 10416
+    },
+    {
+      "epoch": 0.34721483647965967,
+      "grad_norm": 1.5847649574279785,
+      "learning_rate": 3.920007509089102e-05,
+      "loss": 1.4935,
+      "step": 10447
+    },
+    {
+      "epoch": 0.3482451475671364,
+      "grad_norm": 1.3305450677871704,
+      "learning_rate": 3.913087534326357e-05,
+      "loss": 1.506,
+      "step": 10478
+    },
+    {
+      "epoch": 0.3492754586546131,
+      "grad_norm": 1.3079018592834473,
+      "learning_rate": 3.9061516130628475e-05,
+      "loss": 1.5005,
+      "step": 10509
+    },
+    {
+      "epoch": 0.3503057697420899,
+      "grad_norm": 1.3646214008331299,
+      "learning_rate": 3.8991998235695025e-05,
+      "loss": 1.4937,
+      "step": 10540
+    },
+    {
+      "epoch": 0.3513360808295666,
+      "grad_norm": 1.3948808908462524,
+      "learning_rate": 3.8922322442963224e-05,
+      "loss": 1.4737,
+      "step": 10571
+    },
+    {
+      "epoch": 0.35236639191704333,
+      "grad_norm": 1.4026129245758057,
+      "learning_rate": 3.885248953871491e-05,
+      "loss": 1.4873,
+      "step": 10602
+    },
+    {
+      "epoch": 0.3533967030045201,
+      "grad_norm": 1.396984338760376,
+      "learning_rate": 3.8782500311004915e-05,
+      "loss": 1.4958,
+      "step": 10633
+    },
+    {
+      "epoch": 0.3544270140919968,
+      "grad_norm": 1.3693382740020752,
+      "learning_rate": 3.871235554965218e-05,
+      "loss": 1.4886,
+      "step": 10664
+    },
+    {
+      "epoch": 0.35545732517947354,
+      "grad_norm": 1.304598093032837,
+      "learning_rate": 3.864205604623078e-05,
+      "loss": 1.4749,
+      "step": 10695
+    },
+    {
+      "epoch": 0.3564876362669503,
+      "grad_norm": 1.3725641965866089,
+      "learning_rate": 3.857160259406107e-05,
+      "loss": 1.4773,
+      "step": 10726
+    },
+    {
+      "epoch": 0.357517947354427,
+      "grad_norm": 1.37954843044281,
+      "learning_rate": 3.8500995988200674e-05,
+      "loss": 1.5007,
+      "step": 10757
+    },
+    {
+      "epoch": 0.35854825844190374,
+      "grad_norm": 1.7074191570281982,
+      "learning_rate": 3.843023702543556e-05,
+      "loss": 1.488,
+      "step": 10788
+    },
+    {
+      "epoch": 0.3595785695293805,
+      "grad_norm": 1.3874558210372925,
+      "learning_rate": 3.8359326504270984e-05,
+      "loss": 1.4961,
+      "step": 10819
+    },
+    {
+      "epoch": 0.3606088806168572,
+      "grad_norm": 1.4195255041122437,
+      "learning_rate": 3.828826522492255e-05,
+      "loss": 1.4922,
+      "step": 10850
+    },
+    {
+      "epoch": 0.36163919170433395,
+      "grad_norm": 1.3814491033554077,
+      "learning_rate": 3.821705398930713e-05,
+      "loss": 1.4847,
+      "step": 10881
+    },
+    {
+      "epoch": 0.3626695027918107,
+      "grad_norm": 1.269761323928833,
+      "learning_rate": 3.814569360103385e-05,
+      "loss": 1.5009,
+      "step": 10912
+    },
+    {
+      "epoch": 0.3636998138792874,
+      "grad_norm": 1.274278998374939,
+      "learning_rate": 3.807418486539499e-05,
+      "loss": 1.4522,
+      "step": 10943
+    },
+    {
+      "epoch": 0.36473012496676416,
+      "grad_norm": 1.3634709119796753,
+      "learning_rate": 3.80025285893569e-05,
+      "loss": 1.4782,
+      "step": 10974
+    },
+    {
+      "epoch": 0.3657604360542409,
+      "grad_norm": 1.2463103532791138,
+      "learning_rate": 3.793072558155093e-05,
+      "loss": 1.4774,
+      "step": 11005
+    },
+    {
+      "epoch": 0.3667907471417176,
+      "grad_norm": 1.8971500396728516,
+      "learning_rate": 3.785877665226426e-05,
+      "loss": 1.4811,
+      "step": 11036
+    },
+    {
+      "epoch": 0.36782105822919436,
+      "grad_norm": 1.3241803646087646,
+      "learning_rate": 3.778668261343079e-05,
+      "loss": 1.4817,
+      "step": 11067
+    },
+    {
+      "epoch": 0.3688513693166711,
+      "grad_norm": 1.422899842262268,
+      "learning_rate": 3.771444427862192e-05,
+      "loss": 1.4897,
+      "step": 11098
+    },
+    {
+      "epoch": 0.3698816804041478,
+      "grad_norm": 1.3501219749450684,
+      "learning_rate": 3.7642062463037465e-05,
+      "loss": 1.4828,
+      "step": 11129
+    },
+    {
+      "epoch": 0.37091199149162457,
+      "grad_norm": 1.399356722831726,
+      "learning_rate": 3.7569537983496373e-05,
+      "loss": 1.511,
+      "step": 11160
+    },
+    {
+      "epoch": 0.3719423025791013,
+      "grad_norm": 1.358941674232483,
+      "learning_rate": 3.749687165842753e-05,
+      "loss": 1.4887,
+      "step": 11191
+    },
+    {
+      "epoch": 0.372972613666578,
+      "grad_norm": 1.3525429964065552,
+      "learning_rate": 3.7424064307860536e-05,
+      "loss": 1.4929,
+      "step": 11222
+    },
+    {
+      "epoch": 0.3740029247540548,
+      "grad_norm": 1.414780616760254,
+      "learning_rate": 3.735111675341645e-05,
+      "loss": 1.4897,
+      "step": 11253
+    },
+    {
+      "epoch": 0.37503323584153153,
+      "grad_norm": 1.3124057054519653,
+      "learning_rate": 3.7278029818298524e-05,
+      "loss": 1.4697,
+      "step": 11284
+    },
+    {
+      "epoch": 0.37606354692900823,
+      "grad_norm": 1.4073244333267212,
+      "learning_rate": 3.720480432728287e-05,
+      "loss": 1.4857,
+      "step": 11315
+    },
+    {
+      "epoch": 0.377093858016485,
+      "grad_norm": 1.3467692136764526,
+      "learning_rate": 3.71314411067092e-05,
+      "loss": 1.4895,
+      "step": 11346
+    },
+    {
+      "epoch": 0.37812416910396174,
+      "grad_norm": 1.3723987340927124,
+      "learning_rate": 3.70579409844715e-05,
+      "loss": 1.4732,
+      "step": 11377
+    },
+    {
+      "epoch": 0.37915448019143844,
+      "grad_norm": 1.2766269445419312,
+      "learning_rate": 3.698430479000865e-05,
+      "loss": 1.5018,
+      "step": 11408
+    },
+    {
+      "epoch": 0.3801847912789152,
+      "grad_norm": 1.3383387327194214,
+      "learning_rate": 3.691053335429509e-05,
+      "loss": 1.4883,
+      "step": 11439
+    },
+    {
+      "epoch": 0.38121510236639194,
+      "grad_norm": 1.290624737739563,
+      "learning_rate": 3.683662750983147e-05,
+      "loss": 1.464,
+      "step": 11470
+    },
+    {
+      "epoch": 0.38224541345386864,
+      "grad_norm": 1.2905876636505127,
+      "learning_rate": 3.676258809063518e-05,
+      "loss": 1.4854,
+      "step": 11501
+    },
+    {
+      "epoch": 0.3832757245413454,
+      "grad_norm": 1.2959662675857544,
+      "learning_rate": 3.6688415932231004e-05,
+      "loss": 1.4617,
+      "step": 11532
+    },
+    {
+      "epoch": 0.3843060356288221,
+      "grad_norm": 1.4139777421951294,
+      "learning_rate": 3.661411187164166e-05,
+      "loss": 1.4929,
+      "step": 11563
+    },
+    {
+      "epoch": 0.38533634671629885,
+      "grad_norm": 1.387298583984375,
+      "learning_rate": 3.65396767473784e-05,
+      "loss": 1.4792,
+      "step": 11594
+    },
+    {
+      "epoch": 0.3863666578037756,
+      "grad_norm": 1.3097480535507202,
+      "learning_rate": 3.6465111399431465e-05,
+      "loss": 1.4823,
+      "step": 11625
+    },
+    {
+      "epoch": 0.3873969688912523,
+      "grad_norm": 1.3754338026046753,
+      "learning_rate": 3.6390416669260674e-05,
+      "loss": 1.4931,
+      "step": 11656
+    },
+    {
+      "epoch": 0.38842727997872906,
+      "grad_norm": 1.3191930055618286,
+      "learning_rate": 3.63155933997859e-05,
+      "loss": 1.4743,
+      "step": 11687
+    },
+    {
+      "epoch": 0.3894575910662058,
+      "grad_norm": 1.330004096031189,
+      "learning_rate": 3.624064243537758e-05,
+      "loss": 1.4699,
+      "step": 11718
+    },
+    {
+      "epoch": 0.3904879021536825,
+      "grad_norm": 1.286047339439392,
+      "learning_rate": 3.616556462184716e-05,
+      "loss": 1.4773,
+      "step": 11749
+    },
+    {
+      "epoch": 0.39151821324115926,
+      "grad_norm": 1.3343220949172974,
+      "learning_rate": 3.609036080643755e-05,
+      "loss": 1.4798,
+      "step": 11780
+    },
+    {
+      "epoch": 0.392548524328636,
+      "grad_norm": 1.380204677581787,
+      "learning_rate": 3.60150318378136e-05,
+      "loss": 1.4908,
+      "step": 11811
+    },
+    {
+      "epoch": 0.3935788354161127,
+      "grad_norm": 1.290195345878601,
+      "learning_rate": 3.5939578566052465e-05,
+      "loss": 1.4865,
+      "step": 11842
+    },
+    {
+      "epoch": 0.39460914650358947,
+      "grad_norm": 1.3246593475341797,
+      "learning_rate": 3.586400184263408e-05,
+      "loss": 1.4803,
+      "step": 11873
+    },
+    {
+      "epoch": 0.3956394575910662,
+      "grad_norm": 1.2384580373764038,
+      "learning_rate": 3.578830252043148e-05,
+      "loss": 1.4582,
+      "step": 11904
+    },
+    {
+      "epoch": 0.3966697686785429,
+      "grad_norm": 1.3226487636566162,
+      "learning_rate": 3.571248145370125e-05,
+      "loss": 1.4758,
+      "step": 11935
+    },
+    {
+      "epoch": 0.3977000797660197,
+      "grad_norm": 1.3451173305511475,
+      "learning_rate": 3.5636539498073794e-05,
+      "loss": 1.4682,
+      "step": 11966
+    },
+    {
+      "epoch": 0.39873039085349643,
+      "grad_norm": 1.3755093812942505,
+      "learning_rate": 3.556047751054378e-05,
+      "loss": 1.4788,
+      "step": 11997
+    },
+    {
+      "epoch": 0.39976070194097313,
+      "grad_norm": 1.3697580099105835,
+      "learning_rate": 3.548429634946039e-05,
+      "loss": 1.479,
+      "step": 12028
+    },
+    {
+      "epoch": 0.4007910130284499,
+      "grad_norm": 1.2837176322937012,
+      "learning_rate": 3.540799687451768e-05,
+      "loss": 1.4601,
+      "step": 12059
+    },
+    {
+      "epoch": 0.40182132411592664,
+      "grad_norm": 1.2861183881759644,
+      "learning_rate": 3.533157994674485e-05,
+      "loss": 1.4648,
+      "step": 12090
+    },
+    {
+      "epoch": 0.40285163520340334,
+      "grad_norm": 1.3519474267959595,
+      "learning_rate": 3.5255046428496546e-05,
+      "loss": 1.4786,
+      "step": 12121
+    },
+    {
+      "epoch": 0.4038819462908801,
+      "grad_norm": 1.4055910110473633,
+      "learning_rate": 3.517839718344311e-05,
+      "loss": 1.4579,
+      "step": 12152
+    },
+    {
+      "epoch": 0.40491225737835684,
+      "grad_norm": 1.326678991317749,
+      "learning_rate": 3.510163307656086e-05,
+      "loss": 1.4737,
+      "step": 12183
+    },
+    {
+      "epoch": 0.40594256846583354,
+      "grad_norm": 1.2847933769226074,
+      "learning_rate": 3.5024754974122324e-05,
+      "loss": 1.4532,
+      "step": 12214
+    },
+    {
+      "epoch": 0.4069728795533103,
+      "grad_norm": 1.3367128372192383,
+      "learning_rate": 3.494776374368643e-05,
+      "loss": 1.4781,
+      "step": 12245
+    },
+    {
+      "epoch": 0.40800319064078705,
+      "grad_norm": 1.3932790756225586,
+      "learning_rate": 3.4870660254088724e-05,
+      "loss": 1.4752,
+      "step": 12276
+    },
+    {
+      "epoch": 0.40903350172826375,
+      "grad_norm": 1.4312951564788818,
+      "learning_rate": 3.479344537543164e-05,
+      "loss": 1.4842,
+      "step": 12307
+    },
+    {
+      "epoch": 0.4100638128157405,
+      "grad_norm": 1.279990553855896,
+      "learning_rate": 3.4716119979074565e-05,
+      "loss": 1.4747,
+      "step": 12338
+    },
+    {
+      "epoch": 0.41109412390321726,
+      "grad_norm": 1.3121005296707153,
+      "learning_rate": 3.463868493762412e-05,
+      "loss": 1.4848,
+      "step": 12369
+    },
+    {
+      "epoch": 0.41212443499069396,
+      "grad_norm": 1.3275870084762573,
+      "learning_rate": 3.456114112492418e-05,
+      "loss": 1.4617,
+      "step": 12400
+    },
+    {
+      "epoch": 0.4131547460781707,
+      "grad_norm": 1.281539797782898,
+      "learning_rate": 3.4483489416046164e-05,
+      "loss": 1.4757,
+      "step": 12431
+    },
+    {
+      "epoch": 0.4141850571656474,
+      "grad_norm": 1.3381626605987549,
+      "learning_rate": 3.440573068727905e-05,
+      "loss": 1.4757,
+      "step": 12462
+    },
+    {
+      "epoch": 0.41521536825312416,
+      "grad_norm": 1.256617546081543,
+      "learning_rate": 3.4327865816119495e-05,
+      "loss": 1.4512,
+      "step": 12493
+    },
+    {
+      "epoch": 0.4162456793406009,
+      "grad_norm": 1.5587774515151978,
+      "learning_rate": 3.4249895681262025e-05,
+      "loss": 1.4635,
+      "step": 12524
+    },
+    {
+      "epoch": 0.4172759904280776,
+      "grad_norm": 1.2654612064361572,
+      "learning_rate": 3.417182116258899e-05,
+      "loss": 1.4693,
+      "step": 12555
+    },
+    {
+      "epoch": 0.41830630151555437,
+      "grad_norm": 1.4872632026672363,
+      "learning_rate": 3.409364314116074e-05,
+      "loss": 1.4526,
+      "step": 12586
+    },
+    {
+      "epoch": 0.4193366126030311,
+      "grad_norm": 1.263200044631958,
+      "learning_rate": 3.401536249920559e-05,
+      "loss": 1.463,
+      "step": 12617
+    },
+    {
+      "epoch": 0.4203669236905078,
+      "grad_norm": 1.3187345266342163,
+      "learning_rate": 3.393698012010998e-05,
+      "loss": 1.4629,
+      "step": 12648
+    },
+    {
+      "epoch": 0.4213972347779846,
+      "grad_norm": 1.2915629148483276,
+      "learning_rate": 3.385849688840839e-05,
+      "loss": 1.4741,
+      "step": 12679
+    },
+    {
+      "epoch": 0.42242754586546133,
+      "grad_norm": 1.3286950588226318,
+      "learning_rate": 3.3779913689773414e-05,
+      "loss": 1.4625,
+      "step": 12710
+    },
+    {
+      "epoch": 0.423457856952938,
+      "grad_norm": 1.3610268831253052,
+      "learning_rate": 3.370123141100578e-05,
+      "loss": 1.4518,
+      "step": 12741
+    },
+    {
+      "epoch": 0.4244881680404148,
+      "grad_norm": 1.2941581010818481,
+      "learning_rate": 3.3622450940024305e-05,
+      "loss": 1.4727,
+      "step": 12772
+    },
+    {
+      "epoch": 0.42551847912789154,
+      "grad_norm": 1.2699294090270996,
+      "learning_rate": 3.35435731658559e-05,
+      "loss": 1.4516,
+      "step": 12803
+    },
+    {
+      "epoch": 0.42654879021536823,
+      "grad_norm": 1.3749297857284546,
+      "learning_rate": 3.346459897862552e-05,
+      "loss": 1.4949,
+      "step": 12834
+    },
+    {
+      "epoch": 0.427579101302845,
+      "grad_norm": 1.2593483924865723,
+      "learning_rate": 3.338552926954613e-05,
+      "loss": 1.4671,
+      "step": 12865
+    },
+    {
+      "epoch": 0.42860941239032174,
+      "grad_norm": 1.2845182418823242,
+      "learning_rate": 3.330636493090868e-05,
+      "loss": 1.4734,
+      "step": 12896
+    },
+    {
+      "epoch": 0.42963972347779844,
+      "grad_norm": 1.2778210639953613,
+      "learning_rate": 3.322710685607193e-05,
+      "loss": 1.4694,
+      "step": 12927
+    },
+    {
+      "epoch": 0.4306700345652752,
+      "grad_norm": 1.234062671661377,
+      "learning_rate": 3.314775593945251e-05,
+      "loss": 1.4627,
+      "step": 12958
+    },
+    {
+      "epoch": 0.43170034565275195,
+      "grad_norm": 1.360439419746399,
+      "learning_rate": 3.3068313076514714e-05,
+      "loss": 1.4601,
+      "step": 12989
+    },
+    {
+      "epoch": 0.43273065674022865,
+      "grad_norm": 1.2994401454925537,
+      "learning_rate": 3.298877916376047e-05,
+      "loss": 1.4774,
+      "step": 13020
+    },
+    {
+      "epoch": 0.4337609678277054,
+      "grad_norm": 1.3562462329864502,
+      "learning_rate": 3.290915509871915e-05,
+      "loss": 1.461,
+      "step": 13051
+    },
+    {
+      "epoch": 0.43479127891518216,
+      "grad_norm": 1.2676275968551636,
+      "learning_rate": 3.282944177993753e-05,
+      "loss": 1.4676,
+      "step": 13082
+    },
+    {
+      "epoch": 0.43582159000265885,
+      "grad_norm": 1.3941326141357422,
+      "learning_rate": 3.274964010696957e-05,
+      "loss": 1.4673,
+      "step": 13113
+    },
+    {
+      "epoch": 0.4368519010901356,
+      "grad_norm": 1.3286877870559692,
+      "learning_rate": 3.266975098036629e-05,
+      "loss": 1.4759,
+      "step": 13144
+    },
+    {
+      "epoch": 0.43788221217761236,
+      "grad_norm": 1.281602382659912,
+      "learning_rate": 3.258977530166562e-05,
+      "loss": 1.4464,
+      "step": 13175
+    },
+    {
+      "epoch": 0.43891252326508906,
+      "grad_norm": 1.334889531135559,
+      "learning_rate": 3.250971397338227e-05,
+      "loss": 1.4552,
+      "step": 13206
+    },
+    {
+      "epoch": 0.4399428343525658,
+      "grad_norm": 1.2915422916412354,
+      "learning_rate": 3.2429567898997404e-05,
+      "loss": 1.4644,
+      "step": 13237
+    },
+    {
+      "epoch": 0.44097314544004257,
+      "grad_norm": 1.3166664838790894,
+      "learning_rate": 3.234933798294859e-05,
+      "loss": 1.4783,
+      "step": 13268
+    },
+    {
+      "epoch": 0.44200345652751927,
+      "grad_norm": 1.2781778573989868,
+      "learning_rate": 3.2269025130619535e-05,
+      "loss": 1.4658,
+      "step": 13299
+    },
+    {
+      "epoch": 0.443033767614996,
+      "grad_norm": 1.2616426944732666,
+      "learning_rate": 3.218863024832985e-05,
+      "loss": 1.4536,
+      "step": 13330
+    },
+    {
+      "epoch": 0.4440640787024727,
+      "grad_norm": 1.3652106523513794,
+      "learning_rate": 3.2108154243324864e-05,
+      "loss": 1.4479,
+      "step": 13361
+    },
+    {
+      "epoch": 0.4450943897899495,
+      "grad_norm": 1.4294711351394653,
+      "learning_rate": 3.2027598023765345e-05,
+      "loss": 1.4417,
+      "step": 13392
+    },
+    {
+      "epoch": 0.44612470087742623,
+      "grad_norm": 1.617630124092102,
+      "learning_rate": 3.194696249871729e-05,
+      "loss": 1.4418,
+      "step": 13423
+    },
+    {
+      "epoch": 0.4471550119649029,
+      "grad_norm": 1.2971818447113037,
+      "learning_rate": 3.186624857814164e-05,
+      "loss": 1.4527,
+      "step": 13454
+    },
+    {
+      "epoch": 0.4481853230523797,
+      "grad_norm": 1.2804710865020752,
+      "learning_rate": 3.178545717288401e-05,
+      "loss": 1.4568,
+      "step": 13485
+    },
+    {
+      "epoch": 0.44921563413985643,
+      "grad_norm": 1.3158124685287476,
+      "learning_rate": 3.170458919466444e-05,
+      "loss": 1.4808,
+      "step": 13516
+    },
+    {
+      "epoch": 0.45024594522733313,
+      "grad_norm": 1.3273817300796509,
+      "learning_rate": 3.1623645556067063e-05,
+      "loss": 1.4513,
+      "step": 13547
+    },
+    {
+      "epoch": 0.4512762563148099,
+      "grad_norm": 1.3045655488967896,
+      "learning_rate": 3.154262717052985e-05,
+      "loss": 1.4523,
+      "step": 13578
+    },
+    {
+      "epoch": 0.45230656740228664,
+      "grad_norm": 1.2557601928710938,
+      "learning_rate": 3.146153495233426e-05,
+      "loss": 1.4435,
+      "step": 13609
+    },
+    {
+      "epoch": 0.45333687848976334,
+      "grad_norm": 1.3084708452224731,
+      "learning_rate": 3.1380369816594944e-05,
+      "loss": 1.4249,
+      "step": 13640
+    },
+    {
+      "epoch": 0.4543671895772401,
+      "grad_norm": 1.3613477945327759,
+      "learning_rate": 3.129913267924946e-05,
+      "loss": 1.4662,
+      "step": 13671
+    },
+    {
+      "epoch": 0.45539750066471685,
+      "grad_norm": 1.3156840801239014,
+      "learning_rate": 3.121782445704782e-05,
+      "loss": 1.4546,
+      "step": 13702
+    },
+    {
+      "epoch": 0.45642781175219355,
+      "grad_norm": 1.3385698795318604,
+      "learning_rate": 3.11364460675423e-05,
+      "loss": 1.4759,
+      "step": 13733
+    },
+    {
+      "epoch": 0.4574581228396703,
+      "grad_norm": 2.98878812789917,
+      "learning_rate": 3.1054998429076934e-05,
+      "loss": 1.4468,
+      "step": 13764
+    },
+    {
+      "epoch": 0.45848843392714705,
+      "grad_norm": 1.3070143461227417,
+      "learning_rate": 3.097348246077728e-05,
+      "loss": 1.4485,
+      "step": 13795
+    },
+    {
+      "epoch": 0.45951874501462375,
+      "grad_norm": 1.3935726881027222,
+      "learning_rate": 3.0891899082539924e-05,
+      "loss": 1.4608,
+      "step": 13826
+    },
+    {
+      "epoch": 0.4605490561021005,
+      "grad_norm": 1.2395133972167969,
+      "learning_rate": 3.0810249215022233e-05,
+      "loss": 1.4471,
+      "step": 13857
+    },
+    {
+      "epoch": 0.46157936718957726,
+      "grad_norm": 1.2566940784454346,
+      "learning_rate": 3.0728533779631865e-05,
+      "loss": 1.4694,
+      "step": 13888
+    },
+    {
+      "epoch": 0.46260967827705396,
+      "grad_norm": 1.2381232976913452,
+      "learning_rate": 3.064675369851637e-05,
+      "loss": 1.4392,
+      "step": 13919
+    },
+    {
+      "epoch": 0.4636399893645307,
+      "grad_norm": 1.2350660562515259,
+      "learning_rate": 3.056490989455289e-05,
+      "loss": 1.4549,
+      "step": 13950
+    },
+    {
+      "epoch": 0.46467030045200747,
+      "grad_norm": 1.2426815032958984,
+      "learning_rate": 3.0483003291337596e-05,
+      "loss": 1.4489,
+      "step": 13981
+    },
+    {
+      "epoch": 0.46570061153948417,
+      "grad_norm": 1.297097086906433,
+      "learning_rate": 3.040103481317539e-05,
+      "loss": 1.4305,
+      "step": 14012
+    },
+    {
+      "epoch": 0.4667309226269609,
+      "grad_norm": 1.2504866123199463,
+      "learning_rate": 3.03190053850694e-05,
+      "loss": 1.4559,
+      "step": 14043
+    },
+    {
+      "epoch": 0.4677612337144377,
+      "grad_norm": 1.3741912841796875,
+      "learning_rate": 3.0236915932710573e-05,
+      "loss": 1.4495,
+      "step": 14074
+    },
+    {
+      "epoch": 0.4687915448019144,
+      "grad_norm": 1.4412919282913208,
+      "learning_rate": 3.0154767382467232e-05,
+      "loss": 1.4595,
+      "step": 14105
+    },
+    {
+      "epoch": 0.4698218558893911,
+      "grad_norm": 1.2474778890609741,
+      "learning_rate": 3.0072560661374582e-05,
+      "loss": 1.4621,
+      "step": 14136
+    },
+    {
+      "epoch": 0.4708521669768679,
+      "grad_norm": 1.3058266639709473,
+      "learning_rate": 2.999029669712431e-05,
+      "loss": 1.4554,
+      "step": 14167
+    },
+    {
+      "epoch": 0.4718824780643446,
+      "grad_norm": 1.3119689226150513,
+      "learning_rate": 2.990797641805408e-05,
+      "loss": 1.438,
+      "step": 14198
+    },
+    {
+      "epoch": 0.47291278915182133,
+      "grad_norm": 1.3046278953552246,
+      "learning_rate": 2.982560075313704e-05,
+      "loss": 1.4406,
+      "step": 14229
+    },
+    {
+      "epoch": 0.47394310023929803,
+      "grad_norm": 1.3206086158752441,
+      "learning_rate": 2.9743170631971368e-05,
+      "loss": 1.449,
+      "step": 14260
+    },
+    {
+      "epoch": 0.4749734113267748,
+      "grad_norm": 1.3135420083999634,
+      "learning_rate": 2.9660686984769792e-05,
+      "loss": 1.4583,
+      "step": 14291
+    },
+    {
+      "epoch": 0.47600372241425154,
+      "grad_norm": 1.3019664287567139,
+      "learning_rate": 2.9578150742349047e-05,
+      "loss": 1.458,
+      "step": 14322
+    },
+    {
+      "epoch": 0.47703403350172824,
+      "grad_norm": 1.2588508129119873,
+      "learning_rate": 2.949556283611942e-05,
+      "loss": 1.4432,
+      "step": 14353
+    },
+    {
+      "epoch": 0.478064344589205,
+      "grad_norm": 1.2975914478302002,
+      "learning_rate": 2.9412924198074206e-05,
+      "loss": 1.438,
+      "step": 14384
+    },
+    {
+      "epoch": 0.47909465567668175,
+      "grad_norm": 1.2383975982666016,
+      "learning_rate": 2.9330235760779208e-05,
+      "loss": 1.4417,
+      "step": 14415
+    },
+    {
+      "epoch": 0.48012496676415845,
+      "grad_norm": 1.1769133806228638,
+      "learning_rate": 2.9247498457362188e-05,
+      "loss": 1.4509,
+      "step": 14446
+    },
+    {
+      "epoch": 0.4811552778516352,
+      "grad_norm": 1.2747749090194702,
+      "learning_rate": 2.9164713221502373e-05,
+      "loss": 1.4439,
+      "step": 14477
+    },
+    {
+      "epoch": 0.48218558893911195,
+      "grad_norm": 1.2673002481460571,
+      "learning_rate": 2.9081880987419912e-05,
+      "loss": 1.4314,
+      "step": 14508
+    },
+    {
+      "epoch": 0.48321590002658865,
+      "grad_norm": 1.3461123704910278,
+      "learning_rate": 2.8999002689865296e-05,
+      "loss": 1.4203,
+      "step": 14539
+    },
+    {
+      "epoch": 0.4842462111140654,
+      "grad_norm": 1.2993537187576294,
+      "learning_rate": 2.8916079264108852e-05,
+      "loss": 1.4438,
+      "step": 14570
+    },
+    {
+      "epoch": 0.48527652220154216,
+      "grad_norm": 1.2836824655532837,
+      "learning_rate": 2.883311164593017e-05,
+      "loss": 1.4541,
+      "step": 14601
+    },
+    {
+      "epoch": 0.48630683328901886,
+      "grad_norm": 1.220000147819519,
+      "learning_rate": 2.875010077160754e-05,
+      "loss": 1.4473,
+      "step": 14632
+    },
+    {
+      "epoch": 0.4873371443764956,
+      "grad_norm": 1.3283642530441284,
+      "learning_rate": 2.866704757790741e-05,
+      "loss": 1.4581,
+      "step": 14663
+    },
+    {
+      "epoch": 0.48836745546397237,
+      "grad_norm": 1.251988172531128,
+      "learning_rate": 2.858395300207376e-05,
+      "loss": 1.4192,
+      "step": 14694
+    },
+    {
+      "epoch": 0.48939776655144906,
+      "grad_norm": 1.2558730840682983,
+      "learning_rate": 2.8500817981817607e-05,
+      "loss": 1.4569,
+      "step": 14725
+    },
+    {
+      "epoch": 0.4904280776389258,
+      "grad_norm": 1.2405266761779785,
+      "learning_rate": 2.8417643455306336e-05,
+      "loss": 1.4491,
+      "step": 14756
+    },
+    {
+      "epoch": 0.4914583887264026,
+      "grad_norm": 1.3039582967758179,
+      "learning_rate": 2.8334430361153185e-05,
+      "loss": 1.4267,
+      "step": 14787
+    },
+    {
+      "epoch": 0.49248869981387927,
+      "grad_norm": 1.2920899391174316,
+      "learning_rate": 2.8251179638406612e-05,
+      "loss": 1.4565,
+      "step": 14818
+    },
+    {
+      "epoch": 0.493519010901356,
+      "grad_norm": 1.341066837310791,
+      "learning_rate": 2.8167892226539704e-05,
+      "loss": 1.4405,
+      "step": 14849
+    },
+    {
+      "epoch": 0.4945493219888328,
+      "grad_norm": 1.3090940713882446,
+      "learning_rate": 2.8084569065439588e-05,
+      "loss": 1.4339,
+      "step": 14880
+    },
+    {
+      "epoch": 0.4955796330763095,
+      "grad_norm": 1.3220746517181396,
+      "learning_rate": 2.8001211095396807e-05,
+      "loss": 1.4371,
+      "step": 14911
+    },
+    {
+      "epoch": 0.49660994416378623,
+      "grad_norm": 1.2530059814453125,
+      "learning_rate": 2.791781925709473e-05,
+      "loss": 1.4486,
+      "step": 14942
+    },
+    {
+      "epoch": 0.497640255251263,
+      "grad_norm": 1.2478493452072144,
+      "learning_rate": 2.7834394491598908e-05,
+      "loss": 1.4399,
+      "step": 14973
+    },
+    {
+      "epoch": 0.4986705663387397,
+      "grad_norm": 1.3576936721801758,
+      "learning_rate": 2.7750937740346485e-05,
+      "loss": 1.4358,
+      "step": 15004
+    },
+    {
+      "epoch": 0.49970087742621644,
+      "grad_norm": 1.2513341903686523,
+      "learning_rate": 2.7667449945135564e-05,
+      "loss": 1.4549,
+      "step": 15035
+    },
+    {
+      "epoch": 0.5007311885136931,
+      "grad_norm": 1.2199060916900635,
+      "learning_rate": 2.7583932048114557e-05,
+      "loss": 1.4428,
+      "step": 15066
+    },
+    {
+      "epoch": 0.5017614996011699,
+      "grad_norm": 1.3242244720458984,
+      "learning_rate": 2.7500384991771587e-05,
+      "loss": 1.4387,
+      "step": 15097
+    },
+    {
+      "epoch": 0.5027918106886466,
+      "grad_norm": 1.3010181188583374,
+      "learning_rate": 2.7416809718923825e-05,
+      "loss": 1.4352,
+      "step": 15128
+    },
+    {
+      "epoch": 0.5038221217761234,
+      "grad_norm": 1.4653838872909546,
+      "learning_rate": 2.7333207172706864e-05,
+      "loss": 1.4461,
+      "step": 15159
+    },
+    {
+      "epoch": 0.5048524328636002,
+      "grad_norm": 1.2516883611679077,
+      "learning_rate": 2.7249578296564088e-05,
+      "loss": 1.4443,
+      "step": 15190
+    },
+    {
+      "epoch": 0.5058827439510768,
+      "grad_norm": 1.346489667892456,
+      "learning_rate": 2.7165924034235973e-05,
+      "loss": 1.4255,
+      "step": 15221
+    },
+    {
+      "epoch": 0.5069130550385536,
+      "grad_norm": 1.276147723197937,
+      "learning_rate": 2.708224532974953e-05,
+      "loss": 1.4356,
+      "step": 15252
+    }
+  ],
+  "logging_steps": 31,
+  "max_steps": 30517,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 7630,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.1319725270111355e+19,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-15260/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c756a0eeda8352f6a4a5bca7fe984f64ec04e3acd06c2221cd7234ab6cba8f16
+size 5432

checkpoint-22890/config.json

@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "/scratch/dme-llm/ayamaguchi.dme-llm/models/Llama-3.1-8B-ta-madlad-mean/",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.47.0.dev0",
+  "use_cache": true,
+  "vocab_size": 138256
+}

checkpoint-22890/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.47.0.dev0"
+}

checkpoint-22890/model-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8628f9cdff34b61e38190d2a56d5cae595fb16cc62cdcd0a8c6add463de4177
+size 4983197184

checkpoint-22890/model-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b28e50e95c787e68fdf10680635ba27e6d440763a69744dc6ddf473f852231a
+size 4899116432

checkpoint-22890/model-00003-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afa0e22bcade9527b5aeb86066e139cc30979f8492c753ff3277e6189863ab97
+size 4999813112

checkpoint-22890/model-00004-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc4f5352f30ea036b0a4e24ba9b029dd94a2b30a716c81acc1c4a5e7bb47a042
+size 4999813128

checkpoint-22890/model-00005-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d9450b209df4cb026b6669e618fcdb6ff4f5bd0a711fb2e502afe0b95eb3ad7
+size 4832007496

checkpoint-22890/model-00006-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc6e12be5a45414a29f3010843c0601167f380ba0b3f807ad6a2fb2be81a0f15
+size 4999813120

checkpoint-22890/model-00007-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04e27da7adde0beb2c489a7cd25882227ae31dc88894e749a92a3de38c78c590
+size 2734998184

checkpoint-22890/model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 32448724992
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00007-of-00007.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.norm.weight": "model-00007-of-00007.safetensors"
+  }
+}

checkpoint-22890/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c935372459b04e237828c8ea2bf0624792067ae30a8711ac24211bacbd01941c
+size 16040396334

checkpoint-22890/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:386fcc8cc1089aade9450d86fb239ea3483f455fd2d78d8378645feecfec9d69
+size 14244

checkpoint-22890/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9cceae98d85860224f83926a9c2d0797a9f4411abc8a933e66f23226e9ba794a
+size 1064

checkpoint-22890/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c756a0eeda8352f6a4a5bca7fe984f64ec04e3acd06c2221cd7234ab6cba8f16
+size 5432

checkpoint-30517/config.json

@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "/scratch/dme-llm/ayamaguchi.dme-llm/models/Llama-3.1-8B-ta-madlad-mean/",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.47.0.dev0",
+  "use_cache": true,
+  "vocab_size": 138256
+}

checkpoint-30517/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.47.0.dev0"
+}

checkpoint-30517/model-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59f92c66c6043df4728e3cb3931dd679fb0e065750141bf6e3512b2eeda18429
+size 4983197184

checkpoint-30517/model-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b28e50e95c787e68fdf10680635ba27e6d440763a69744dc6ddf473f852231a
+size 4899116432

checkpoint-30517/model-00003-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afa0e22bcade9527b5aeb86066e139cc30979f8492c753ff3277e6189863ab97
+size 4999813112

checkpoint-30517/model-00004-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc4f5352f30ea036b0a4e24ba9b029dd94a2b30a716c81acc1c4a5e7bb47a042
+size 4999813128

checkpoint-30517/model-00005-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d9450b209df4cb026b6669e618fcdb6ff4f5bd0a711fb2e502afe0b95eb3ad7
+size 4832007496

checkpoint-30517/model-00006-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4762ff5824a3e9dc76ef64c04b0e9e8cd5e84b26ec4fcb4b0cf20f17917f497e
+size 4999813120

checkpoint-30517/model-00007-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74c3c7a4a6d6cab9de7f753a370e71637ffbb150bbf0cf0f0dd808362ba5aff9
+size 2734998184

checkpoint-30517/model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 32448724992
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00007-of-00007.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.norm.weight": "model-00007-of-00007.safetensors"
+  }
+}

checkpoint-30517/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70fc066e87f5ea7bd0fc6411fdc2ec0fa3ca8e8f19604300f4baee58e1391ed1
+size 16040396334

checkpoint-30517/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ff264f99d31b522cc7e2a4eac9d38606d0c58a34c0adc74d71e0ca8b371dc36
+size 14244

checkpoint-30517/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d91c597306f26392e8da0d33a09fdcca77cfdeb5ad42248045521772fa3d64e
+size 1064

checkpoint-30517/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c756a0eeda8352f6a4a5bca7fe984f64ec04e3acd06c2221cd7234ab6cba8f16
+size 5432

checkpoint-7630/config.json

@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "/scratch/dme-llm/ayamaguchi.dme-llm/models/Llama-3.1-8B-ta-madlad-mean/",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.47.0.dev0",
+  "use_cache": true,
+  "vocab_size": 138256
+}

checkpoint-7630/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.47.0.dev0"
+}

checkpoint-7630/model-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66d1d3aedc8f6507752a84da24bcffe8c74ce174b9400f28a8dbd088538a022b
+size 4983197184

checkpoint-7630/model-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b28e50e95c787e68fdf10680635ba27e6d440763a69744dc6ddf473f852231a
+size 4899116432

checkpoint-7630/model-00003-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afa0e22bcade9527b5aeb86066e139cc30979f8492c753ff3277e6189863ab97
+size 4999813112