Upload folder using huggingface_hub

added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

checkpoint-15260/config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "Qwen/Qwen2.5-7B",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 131072,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.0.dev0",
+  "use_cache": true,
+  "use_mrope": false,
+  "use_sliding_window": false,
+  "vocab_size": 152064
+}

checkpoint-15260/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "max_new_tokens": 2048,
+  "transformers_version": "4.44.0.dev0"
+}

checkpoint-15260/model-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8d0bba72f6cce0f59d7b80ada646efcd18941ce64d10d6fac484e321dc4bc41
+size 4976687216

checkpoint-15260/model-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:faab92c079dee70573da2aa38bf83c1a2a2858fbee7104c30f77e30eb62ed67f
+size 4778622352

checkpoint-15260/model-00003-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d579560f2723d4a62b22ef6ec0d935dfbc2ddf0f14fc9efeac83bf5b586bf9d6
+size 4932743960

checkpoint-15260/model-00004-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:089cf99962fd351ceb590220ea351f1d18937b35376077982b276416f768317a
+size 4932743992

checkpoint-15260/model-00005-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b61bee4a8b5e41f744ec0820def58c39c0c333e6787b7e24146468348f082a0
+size 4998852296

checkpoint-15260/model-00006-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ea7e5ad73a68ac3aa30a5692a531e642396a2870664e3933473c16ccd666cb6
+size 3662865184

checkpoint-15260/model-00007-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2522331f1b0ed7b030b7069d1d861e9b84f95ae4c15dd324d2f051ef19e91f7
+size 2179989632

checkpoint-15260/model.safetensors.index.json

@@ -0,0 +1,346 @@
+{
+  "metadata": {
+    "total_size": 30462466048
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00007-of-00007.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.norm.weight": "model-00006-of-00007.safetensors"
+  }
+}

checkpoint-15260/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67239b090db9514d0762098cbe7b7259fcedabf09571cc124c2964273553b75d
+size 16177880918

checkpoint-15260/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
+size 14244

checkpoint-15260/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ce335347efaebf038b8915bc0a8d2d587a4a1aa08ad0e015b4bc7cc4fba634e
+size 1064

checkpoint-15260/trainer_state.json

@@ -0,0 +1,3477 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.250336297122609,
+  "eval_steps": 500,
+  "global_step": 15260,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0005085468683355753,
+      "grad_norm": 1.63396155834198,
+      "learning_rate": 1.0157273918741808e-06,
+      "loss": 1.1247,
+      "step": 31
+    },
+    {
+      "epoch": 0.0010170937366711507,
+      "grad_norm": 1.4367411136627197,
+      "learning_rate": 2.0314547837483616e-06,
+      "loss": 1.0996,
+      "step": 62
+    },
+    {
+      "epoch": 0.001525640605006726,
+      "grad_norm": 1.6043877601623535,
+      "learning_rate": 3.0471821756225426e-06,
+      "loss": 1.077,
+      "step": 93
+    },
+    {
+      "epoch": 0.0020341874733423014,
+      "grad_norm": 1.5942660570144653,
+      "learning_rate": 4.062909567496723e-06,
+      "loss": 1.0578,
+      "step": 124
+    },
+    {
+      "epoch": 0.0025427343416778766,
+      "grad_norm": 1.5336261987686157,
+      "learning_rate": 5.078636959370905e-06,
+      "loss": 1.0565,
+      "step": 155
+    },
+    {
+      "epoch": 0.003051281210013452,
+      "grad_norm": 1.4558008909225464,
+      "learning_rate": 6.094364351245085e-06,
+      "loss": 1.05,
+      "step": 186
+    },
+    {
+      "epoch": 0.003559828078349027,
+      "grad_norm": 1.5234432220458984,
+      "learning_rate": 7.110091743119267e-06,
+      "loss": 1.02,
+      "step": 217
+    },
+    {
+      "epoch": 0.004068374946684603,
+      "grad_norm": 1.5161484479904175,
+      "learning_rate": 8.125819134993446e-06,
+      "loss": 1.0395,
+      "step": 248
+    },
+    {
+      "epoch": 0.0045769218150201775,
+      "grad_norm": 1.5559700727462769,
+      "learning_rate": 9.141546526867629e-06,
+      "loss": 1.022,
+      "step": 279
+    },
+    {
+      "epoch": 0.005085468683355753,
+      "grad_norm": 1.5957224369049072,
+      "learning_rate": 1.015727391874181e-05,
+      "loss": 1.0274,
+      "step": 310
+    },
+    {
+      "epoch": 0.005594015551691329,
+      "grad_norm": 1.7991306781768799,
+      "learning_rate": 1.117300131061599e-05,
+      "loss": 1.0165,
+      "step": 341
+    },
+    {
+      "epoch": 0.006102562420026904,
+      "grad_norm": 1.5399068593978882,
+      "learning_rate": 1.218872870249017e-05,
+      "loss": 1.0147,
+      "step": 372
+    },
+    {
+      "epoch": 0.006611109288362479,
+      "grad_norm": 1.6368353366851807,
+      "learning_rate": 1.3204456094364351e-05,
+      "loss": 1.0279,
+      "step": 403
+    },
+    {
+      "epoch": 0.007119656156698054,
+      "grad_norm": 1.8204905986785889,
+      "learning_rate": 1.4220183486238533e-05,
+      "loss": 1.0059,
+      "step": 434
+    },
+    {
+      "epoch": 0.00762820302503363,
+      "grad_norm": 1.5241609811782837,
+      "learning_rate": 1.5235910878112714e-05,
+      "loss": 1.0033,
+      "step": 465
+    },
+    {
+      "epoch": 0.008136749893369205,
+      "grad_norm": 1.724233865737915,
+      "learning_rate": 1.6251638269986893e-05,
+      "loss": 1.0042,
+      "step": 496
+    },
+    {
+      "epoch": 0.008645296761704781,
+      "grad_norm": 1.6719369888305664,
+      "learning_rate": 1.7267365661861077e-05,
+      "loss": 1.0024,
+      "step": 527
+    },
+    {
+      "epoch": 0.009153843630040355,
+      "grad_norm": 1.5160149335861206,
+      "learning_rate": 1.8283093053735257e-05,
+      "loss": 1.0037,
+      "step": 558
+    },
+    {
+      "epoch": 0.00966239049837593,
+      "grad_norm": 1.6364191770553589,
+      "learning_rate": 1.9298820445609438e-05,
+      "loss": 0.9964,
+      "step": 589
+    },
+    {
+      "epoch": 0.010170937366711506,
+      "grad_norm": 1.6187313795089722,
+      "learning_rate": 2.031454783748362e-05,
+      "loss": 0.9883,
+      "step": 620
+    },
+    {
+      "epoch": 0.010679484235047082,
+      "grad_norm": 1.5778186321258545,
+      "learning_rate": 2.13302752293578e-05,
+      "loss": 0.9923,
+      "step": 651
+    },
+    {
+      "epoch": 0.011188031103382658,
+      "grad_norm": 1.7242299318313599,
+      "learning_rate": 2.234600262123198e-05,
+      "loss": 0.9809,
+      "step": 682
+    },
+    {
+      "epoch": 0.011696577971718232,
+      "grad_norm": 1.5848108530044556,
+      "learning_rate": 2.336173001310616e-05,
+      "loss": 0.9885,
+      "step": 713
+    },
+    {
+      "epoch": 0.012205124840053807,
+      "grad_norm": 1.5478544235229492,
+      "learning_rate": 2.437745740498034e-05,
+      "loss": 0.9893,
+      "step": 744
+    },
+    {
+      "epoch": 0.012713671708389383,
+      "grad_norm": 1.4508167505264282,
+      "learning_rate": 2.5393184796854525e-05,
+      "loss": 0.9843,
+      "step": 775
+    },
+    {
+      "epoch": 0.013222218576724959,
+      "grad_norm": 1.3082153797149658,
+      "learning_rate": 2.6408912188728702e-05,
+      "loss": 0.9858,
+      "step": 806
+    },
+    {
+      "epoch": 0.013730765445060534,
+      "grad_norm": 1.8803132772445679,
+      "learning_rate": 2.7424639580602886e-05,
+      "loss": 0.9835,
+      "step": 837
+    },
+    {
+      "epoch": 0.014239312313396108,
+      "grad_norm": 1.439850926399231,
+      "learning_rate": 2.8440366972477066e-05,
+      "loss": 0.978,
+      "step": 868
+    },
+    {
+      "epoch": 0.014747859181731684,
+      "grad_norm": 1.5672752857208252,
+      "learning_rate": 2.9456094364351244e-05,
+      "loss": 0.9775,
+      "step": 899
+    },
+    {
+      "epoch": 0.01525640605006726,
+      "grad_norm": 1.4297682046890259,
+      "learning_rate": 3.0471821756225428e-05,
+      "loss": 0.9833,
+      "step": 930
+    },
+    {
+      "epoch": 0.015764952918402835,
+      "grad_norm": 1.6737698316574097,
+      "learning_rate": 3.148754914809961e-05,
+      "loss": 0.9738,
+      "step": 961
+    },
+    {
+      "epoch": 0.01627349978673841,
+      "grad_norm": 1.5833110809326172,
+      "learning_rate": 3.2503276539973785e-05,
+      "loss": 0.9701,
+      "step": 992
+    },
+    {
+      "epoch": 0.016782046655073986,
+      "grad_norm": 1.3715567588806152,
+      "learning_rate": 3.351900393184797e-05,
+      "loss": 0.9697,
+      "step": 1023
+    },
+    {
+      "epoch": 0.017290593523409562,
+      "grad_norm": 1.5785913467407227,
+      "learning_rate": 3.453473132372215e-05,
+      "loss": 0.9672,
+      "step": 1054
+    },
+    {
+      "epoch": 0.017799140391745134,
+      "grad_norm": 1.4025524854660034,
+      "learning_rate": 3.555045871559633e-05,
+      "loss": 0.9663,
+      "step": 1085
+    },
+    {
+      "epoch": 0.01830768726008071,
+      "grad_norm": 1.395158290863037,
+      "learning_rate": 3.6566186107470514e-05,
+      "loss": 0.965,
+      "step": 1116
+    },
+    {
+      "epoch": 0.018816234128416286,
+      "grad_norm": 1.4200639724731445,
+      "learning_rate": 3.7581913499344695e-05,
+      "loss": 0.9572,
+      "step": 1147
+    },
+    {
+      "epoch": 0.01932478099675186,
+      "grad_norm": 1.52960205078125,
+      "learning_rate": 3.8597640891218876e-05,
+      "loss": 0.9512,
+      "step": 1178
+    },
+    {
+      "epoch": 0.019833327865087437,
+      "grad_norm": 1.1526886224746704,
+      "learning_rate": 3.9613368283093056e-05,
+      "loss": 0.9601,
+      "step": 1209
+    },
+    {
+      "epoch": 0.020341874733423013,
+      "grad_norm": 1.2839049100875854,
+      "learning_rate": 4.062909567496724e-05,
+      "loss": 0.9613,
+      "step": 1240
+    },
+    {
+      "epoch": 0.02085042160175859,
+      "grad_norm": 1.4190101623535156,
+      "learning_rate": 4.164482306684142e-05,
+      "loss": 0.9685,
+      "step": 1271
+    },
+    {
+      "epoch": 0.021358968470094164,
+      "grad_norm": 1.146508812904358,
+      "learning_rate": 4.26605504587156e-05,
+      "loss": 0.9675,
+      "step": 1302
+    },
+    {
+      "epoch": 0.02186751533842974,
+      "grad_norm": 1.3196191787719727,
+      "learning_rate": 4.367627785058978e-05,
+      "loss": 0.9544,
+      "step": 1333
+    },
+    {
+      "epoch": 0.022376062206765315,
+      "grad_norm": 1.4939711093902588,
+      "learning_rate": 4.469200524246396e-05,
+      "loss": 0.9546,
+      "step": 1364
+    },
+    {
+      "epoch": 0.022884609075100888,
+      "grad_norm": 1.3225727081298828,
+      "learning_rate": 4.570773263433814e-05,
+      "loss": 0.9603,
+      "step": 1395
+    },
+    {
+      "epoch": 0.023393155943436463,
+      "grad_norm": 1.3572574853897095,
+      "learning_rate": 4.672346002621232e-05,
+      "loss": 0.9535,
+      "step": 1426
+    },
+    {
+      "epoch": 0.02390170281177204,
+      "grad_norm": 1.6103112697601318,
+      "learning_rate": 4.77391874180865e-05,
+      "loss": 0.9596,
+      "step": 1457
+    },
+    {
+      "epoch": 0.024410249680107615,
+      "grad_norm": 1.2908817529678345,
+      "learning_rate": 4.875491480996068e-05,
+      "loss": 0.9414,
+      "step": 1488
+    },
+    {
+      "epoch": 0.02491879654844319,
+      "grad_norm": 1.4638068675994873,
+      "learning_rate": 4.977064220183487e-05,
+      "loss": 0.9554,
+      "step": 1519
+    },
+    {
+      "epoch": 0.025427343416778766,
+      "grad_norm": 1.073677897453308,
+      "learning_rate": 4.9999915451558777e-05,
+      "loss": 0.9424,
+      "step": 1550
+    },
+    {
+      "epoch": 0.02593589028511434,
+      "grad_norm": 1.3571419715881348,
+      "learning_rate": 4.999955597496219e-05,
+      "loss": 0.9397,
+      "step": 1581
+    },
+    {
+      "epoch": 0.026444437153449917,
+      "grad_norm": 1.3831264972686768,
+      "learning_rate": 4.9998914381774255e-05,
+      "loss": 0.9403,
+      "step": 1612
+    },
+    {
+      "epoch": 0.026952984021785493,
+      "grad_norm": 1.3410816192626953,
+      "learning_rate": 4.999799067923527e-05,
+      "loss": 0.939,
+      "step": 1643
+    },
+    {
+      "epoch": 0.02746153089012107,
+      "grad_norm": 1.507271647453308,
+      "learning_rate": 4.999678487776908e-05,
+      "loss": 0.9543,
+      "step": 1674
+    },
+    {
+      "epoch": 0.02797007775845664,
+      "grad_norm": 1.2476454973220825,
+      "learning_rate": 4.9995296990983006e-05,
+      "loss": 0.9391,
+      "step": 1705
+    },
+    {
+      "epoch": 0.028478624626792216,
+      "grad_norm": 1.167040228843689,
+      "learning_rate": 4.999352703566763e-05,
+      "loss": 0.9497,
+      "step": 1736
+    },
+    {
+      "epoch": 0.028987171495127792,
+      "grad_norm": 1.2364312410354614,
+      "learning_rate": 4.999147503179668e-05,
+      "loss": 0.9475,
+      "step": 1767
+    },
+    {
+      "epoch": 0.029495718363463368,
+      "grad_norm": 1.216690182685852,
+      "learning_rate": 4.998914100252672e-05,
+      "loss": 0.9279,
+      "step": 1798
+    },
+    {
+      "epoch": 0.030004265231798943,
+      "grad_norm": 1.1088354587554932,
+      "learning_rate": 4.998652497419696e-05,
+      "loss": 0.9402,
+      "step": 1829
+    },
+    {
+      "epoch": 0.03051281210013452,
+      "grad_norm": 1.261315107345581,
+      "learning_rate": 4.9983626976328927e-05,
+      "loss": 0.9295,
+      "step": 1860
+    },
+    {
+      "epoch": 0.031021358968470095,
+      "grad_norm": 1.0668714046478271,
+      "learning_rate": 4.998044704162613e-05,
+      "loss": 0.9341,
+      "step": 1891
+    },
+    {
+      "epoch": 0.03152990583680567,
+      "grad_norm": 1.2189582586288452,
+      "learning_rate": 4.9976985205973705e-05,
+      "loss": 0.9257,
+      "step": 1922
+    },
+    {
+      "epoch": 0.032038452705141246,
+      "grad_norm": 1.318808913230896,
+      "learning_rate": 4.997324150843799e-05,
+      "loss": 0.9243,
+      "step": 1953
+    },
+    {
+      "epoch": 0.03254699957347682,
+      "grad_norm": 1.0546144247055054,
+      "learning_rate": 4.99692159912661e-05,
+      "loss": 0.9223,
+      "step": 1984
+    },
+    {
+      "epoch": 0.0330555464418124,
+      "grad_norm": 1.2096277475357056,
+      "learning_rate": 4.996490869988546e-05,
+      "loss": 0.9232,
+      "step": 2015
+    },
+    {
+      "epoch": 0.03356409331014797,
+      "grad_norm": 1.1482577323913574,
+      "learning_rate": 4.996031968290326e-05,
+      "loss": 0.9218,
+      "step": 2046
+    },
+    {
+      "epoch": 0.03407264017848355,
+      "grad_norm": 1.1718021631240845,
+      "learning_rate": 4.995544899210594e-05,
+      "loss": 0.9292,
+      "step": 2077
+    },
+    {
+      "epoch": 0.034581187046819124,
+      "grad_norm": 1.2143489122390747,
+      "learning_rate": 4.9950296682458583e-05,
+      "loss": 0.9255,
+      "step": 2108
+    },
+    {
+      "epoch": 0.0350897339151547,
+      "grad_norm": 1.1793973445892334,
+      "learning_rate": 4.994486281210429e-05,
+      "loss": 0.9281,
+      "step": 2139
+    },
+    {
+      "epoch": 0.03559828078349027,
+      "grad_norm": 1.1510167121887207,
+      "learning_rate": 4.9939147442363566e-05,
+      "loss": 0.9226,
+      "step": 2170
+    },
+    {
+      "epoch": 0.036106827651825844,
+      "grad_norm": 1.4281586408615112,
+      "learning_rate": 4.9933150637733574e-05,
+      "loss": 0.9446,
+      "step": 2201
+    },
+    {
+      "epoch": 0.03661537452016142,
+      "grad_norm": 1.2391124963760376,
+      "learning_rate": 4.992687246588743e-05,
+      "loss": 0.9255,
+      "step": 2232
+    },
+    {
+      "epoch": 0.037123921388496996,
+      "grad_norm": 1.3725296258926392,
+      "learning_rate": 4.992031299767347e-05,
+      "loss": 0.9303,
+      "step": 2263
+    },
+    {
+      "epoch": 0.03763246825683257,
+      "grad_norm": 1.009924292564392,
+      "learning_rate": 4.9913472307114386e-05,
+      "loss": 0.9282,
+      "step": 2294
+    },
+    {
+      "epoch": 0.03814101512516815,
+      "grad_norm": 1.247854232788086,
+      "learning_rate": 4.9906350471406446e-05,
+      "loss": 0.9456,
+      "step": 2325
+    },
+    {
+      "epoch": 0.03864956199350372,
+      "grad_norm": 1.237513780593872,
+      "learning_rate": 4.989894757091861e-05,
+      "loss": 0.9494,
+      "step": 2356
+    },
+    {
+      "epoch": 0.0391581088618393,
+      "grad_norm": 1.432076334953308,
+      "learning_rate": 4.989126368919158e-05,
+      "loss": 0.9448,
+      "step": 2387
+    },
+    {
+      "epoch": 0.039666655730174874,
+      "grad_norm": 1.1404424905776978,
+      "learning_rate": 4.988329891293693e-05,
+      "loss": 0.9274,
+      "step": 2418
+    },
+    {
+      "epoch": 0.04017520259851045,
+      "grad_norm": 1.3402444124221802,
+      "learning_rate": 4.987505333203608e-05,
+      "loss": 0.9305,
+      "step": 2449
+    },
+    {
+      "epoch": 0.040683749466846025,
+      "grad_norm": 1.5265156030654907,
+      "learning_rate": 4.9866527039539276e-05,
+      "loss": 0.9306,
+      "step": 2480
+    },
+    {
+      "epoch": 0.0411922963351816,
+      "grad_norm": 1.2100895643234253,
+      "learning_rate": 4.9857720131664594e-05,
+      "loss": 0.9234,
+      "step": 2511
+    },
+    {
+      "epoch": 0.04170084320351718,
+      "grad_norm": 1.294089674949646,
+      "learning_rate": 4.9848632707796773e-05,
+      "loss": 0.9217,
+      "step": 2542
+    },
+    {
+      "epoch": 0.04220939007185275,
+      "grad_norm": 1.159820556640625,
+      "learning_rate": 4.9839264870486155e-05,
+      "loss": 0.9122,
+      "step": 2573
+    },
+    {
+      "epoch": 0.04271793694018833,
+      "grad_norm": 1.366947054862976,
+      "learning_rate": 4.9829616725447526e-05,
+      "loss": 0.9248,
+      "step": 2604
+    },
+    {
+      "epoch": 0.043226483808523904,
+      "grad_norm": 1.3544247150421143,
+      "learning_rate": 4.981968838155888e-05,
+      "loss": 0.9096,
+      "step": 2635
+    },
+    {
+      "epoch": 0.04373503067685948,
+      "grad_norm": 1.1079386472702026,
+      "learning_rate": 4.980947995086024e-05,
+      "loss": 0.9148,
+      "step": 2666
+    },
+    {
+      "epoch": 0.044243577545195055,
+      "grad_norm": 1.1194462776184082,
+      "learning_rate": 4.979899154855234e-05,
+      "loss": 0.9092,
+      "step": 2697
+    },
+    {
+      "epoch": 0.04475212441353063,
+      "grad_norm": 0.9909219741821289,
+      "learning_rate": 4.9788223292995386e-05,
+      "loss": 0.9063,
+      "step": 2728
+    },
+    {
+      "epoch": 0.045260671281866206,
+      "grad_norm": 1.0228219032287598,
+      "learning_rate": 4.977717530570768e-05,
+      "loss": 0.9055,
+      "step": 2759
+    },
+    {
+      "epoch": 0.045769218150201775,
+      "grad_norm": 1.0451685190200806,
+      "learning_rate": 4.976584771136425e-05,
+      "loss": 0.9043,
+      "step": 2790
+    },
+    {
+      "epoch": 0.04627776501853735,
+      "grad_norm": 1.0479145050048828,
+      "learning_rate": 4.975424063779547e-05,
+      "loss": 0.8982,
+      "step": 2821
+    },
+    {
+      "epoch": 0.046786311886872926,
+      "grad_norm": 0.9020496010780334,
+      "learning_rate": 4.974235421598557e-05,
+      "loss": 0.9177,
+      "step": 2852
+    },
+    {
+      "epoch": 0.0472948587552085,
+      "grad_norm": 1.1106877326965332,
+      "learning_rate": 4.973018858007122e-05,
+      "loss": 0.9038,
+      "step": 2883
+    },
+    {
+      "epoch": 0.04780340562354408,
+      "grad_norm": 1.1628150939941406,
+      "learning_rate": 4.9717743867339963e-05,
+      "loss": 0.9014,
+      "step": 2914
+    },
+    {
+      "epoch": 0.04831195249187965,
+      "grad_norm": 0.8586986064910889,
+      "learning_rate": 4.9705020218228695e-05,
+      "loss": 0.9084,
+      "step": 2945
+    },
+    {
+      "epoch": 0.04882049936021523,
+      "grad_norm": 1.1424506902694702,
+      "learning_rate": 4.969201777632205e-05,
+      "loss": 0.9097,
+      "step": 2976
+    },
+    {
+      "epoch": 0.049329046228550805,
+      "grad_norm": 1.049383521080017,
+      "learning_rate": 4.9678736688350846e-05,
+      "loss": 0.9101,
+      "step": 3007
+    },
+    {
+      "epoch": 0.04983759309688638,
+      "grad_norm": 1.0716655254364014,
+      "learning_rate": 4.966517710419033e-05,
+      "loss": 0.8942,
+      "step": 3038
+    },
+    {
+      "epoch": 0.050346139965221956,
+      "grad_norm": 1.2842979431152344,
+      "learning_rate": 4.965133917685858e-05,
+      "loss": 0.893,
+      "step": 3069
+    },
+    {
+      "epoch": 0.05085468683355753,
+      "grad_norm": 1.2429577112197876,
+      "learning_rate": 4.9637223062514714e-05,
+      "loss": 0.9048,
+      "step": 3100
+    },
+    {
+      "epoch": 0.05136323370189311,
+      "grad_norm": 1.1700332164764404,
+      "learning_rate": 4.962282892045718e-05,
+      "loss": 0.901,
+      "step": 3131
+    },
+    {
+      "epoch": 0.05187178057022868,
+      "grad_norm": 0.9648764133453369,
+      "learning_rate": 4.9608156913121904e-05,
+      "loss": 0.8952,
+      "step": 3162
+    },
+    {
+      "epoch": 0.05238032743856426,
+      "grad_norm": 0.9761034846305847,
+      "learning_rate": 4.959320720608049e-05,
+      "loss": 0.8986,
+      "step": 3193
+    },
+    {
+      "epoch": 0.052888874306899834,
+      "grad_norm": 1.5088787078857422,
+      "learning_rate": 4.9577979968038354e-05,
+      "loss": 0.8946,
+      "step": 3224
+    },
+    {
+      "epoch": 0.05339742117523541,
+      "grad_norm": 1.146229863166809,
+      "learning_rate": 4.956247537083282e-05,
+      "loss": 0.8974,
+      "step": 3255
+    },
+    {
+      "epoch": 0.053905968043570986,
+      "grad_norm": 1.002907395362854,
+      "learning_rate": 4.9546693589431145e-05,
+      "loss": 0.8899,
+      "step": 3286
+    },
+    {
+      "epoch": 0.05441451491190656,
+      "grad_norm": 1.2740410566329956,
+      "learning_rate": 4.9530634801928595e-05,
+      "loss": 0.9018,
+      "step": 3317
+    },
+    {
+      "epoch": 0.05492306178024214,
+      "grad_norm": 1.2674225568771362,
+      "learning_rate": 4.9514299189546395e-05,
+      "loss": 0.8887,
+      "step": 3348
+    },
+    {
+      "epoch": 0.05543160864857771,
+      "grad_norm": 1.0592577457427979,
+      "learning_rate": 4.949768693662973e-05,
+      "loss": 0.8913,
+      "step": 3379
+    },
+    {
+      "epoch": 0.05594015551691328,
+      "grad_norm": 1.102695345878601,
+      "learning_rate": 4.948079823064559e-05,
+      "loss": 0.9025,
+      "step": 3410
+    },
+    {
+      "epoch": 0.05644870238524886,
+      "grad_norm": 0.8819742798805237,
+      "learning_rate": 4.946363326218074e-05,
+      "loss": 0.9062,
+      "step": 3441
+    },
+    {
+      "epoch": 0.05695724925358443,
+      "grad_norm": 0.9267928600311279,
+      "learning_rate": 4.9446192224939525e-05,
+      "loss": 0.8853,
+      "step": 3472
+    },
+    {
+      "epoch": 0.05746579612192001,
+      "grad_norm": 1.0881078243255615,
+      "learning_rate": 4.942847531574167e-05,
+      "loss": 0.8845,
+      "step": 3503
+    },
+    {
+      "epoch": 0.057974342990255584,
+      "grad_norm": 1.3232954740524292,
+      "learning_rate": 4.941048273452008e-05,
+      "loss": 0.8968,
+      "step": 3534
+    },
+    {
+      "epoch": 0.05848288985859116,
+      "grad_norm": 4.223265171051025,
+      "learning_rate": 4.9392214684318605e-05,
+      "loss": 0.8826,
+      "step": 3565
+    },
+    {
+      "epoch": 0.058991436726926735,
+      "grad_norm": 1.0562529563903809,
+      "learning_rate": 4.93736713712897e-05,
+      "loss": 0.895,
+      "step": 3596
+    },
+    {
+      "epoch": 0.05949998359526231,
+      "grad_norm": 4.581043720245361,
+      "learning_rate": 4.9354853004692124e-05,
+      "loss": 0.8889,
+      "step": 3627
+    },
+    {
+      "epoch": 0.06000853046359789,
+      "grad_norm": 1.0798181295394897,
+      "learning_rate": 4.93357597968886e-05,
+      "loss": 0.8866,
+      "step": 3658
+    },
+    {
+      "epoch": 0.06051707733193346,
+      "grad_norm": 1.0834298133850098,
+      "learning_rate": 4.931639196334338e-05,
+      "loss": 0.8876,
+      "step": 3689
+    },
+    {
+      "epoch": 0.06102562420026904,
+      "grad_norm": 0.8777247071266174,
+      "learning_rate": 4.9296749722619826e-05,
+      "loss": 0.8923,
+      "step": 3720
+    },
+    {
+      "epoch": 0.061534171068604614,
+      "grad_norm": 0.9053150415420532,
+      "learning_rate": 4.9276833296377966e-05,
+      "loss": 0.8752,
+      "step": 3751
+    },
+    {
+      "epoch": 0.06204271793694019,
+      "grad_norm": 0.9765118956565857,
+      "learning_rate": 4.925664290937196e-05,
+      "loss": 0.8714,
+      "step": 3782
+    },
+    {
+      "epoch": 0.06255126480527576,
+      "grad_norm": 0.927383542060852,
+      "learning_rate": 4.9236178789447576e-05,
+      "loss": 0.882,
+      "step": 3813
+    },
+    {
+      "epoch": 0.06305981167361134,
+      "grad_norm": 1.1646597385406494,
+      "learning_rate": 4.921544116753962e-05,
+      "loss": 0.8811,
+      "step": 3844
+    },
+    {
+      "epoch": 0.06356835854194691,
+      "grad_norm": 0.9912064671516418,
+      "learning_rate": 4.919443027766935e-05,
+      "loss": 0.8807,
+      "step": 3875
+    },
+    {
+      "epoch": 0.06407690541028249,
+      "grad_norm": 1.114752173423767,
+      "learning_rate": 4.91731463569418e-05,
+      "loss": 0.8937,
+      "step": 3906
+    },
+    {
+      "epoch": 0.06458545227861806,
+      "grad_norm": 1.0039126873016357,
+      "learning_rate": 4.915158964554312e-05,
+      "loss": 0.8744,
+      "step": 3937
+    },
+    {
+      "epoch": 0.06509399914695364,
+      "grad_norm": 0.9225612878799438,
+      "learning_rate": 4.912976038673786e-05,
+      "loss": 0.8785,
+      "step": 3968
+    },
+    {
+      "epoch": 0.06560254601528921,
+      "grad_norm": 0.921144425868988,
+      "learning_rate": 4.9107658826866254e-05,
+      "loss": 0.8833,
+      "step": 3999
+    },
+    {
+      "epoch": 0.0661110928836248,
+      "grad_norm": 1.1225559711456299,
+      "learning_rate": 4.908528521534139e-05,
+      "loss": 0.8827,
+      "step": 4030
+    },
+    {
+      "epoch": 0.06661963975196036,
+      "grad_norm": 0.9798495173454285,
+      "learning_rate": 4.906263980464644e-05,
+      "loss": 0.8728,
+      "step": 4061
+    },
+    {
+      "epoch": 0.06712818662029595,
+      "grad_norm": 0.8692976236343384,
+      "learning_rate": 4.903972285033178e-05,
+      "loss": 0.8792,
+      "step": 4092
+    },
+    {
+      "epoch": 0.06763673348863151,
+      "grad_norm": 0.9348081350326538,
+      "learning_rate": 4.901653461101213e-05,
+      "loss": 0.875,
+      "step": 4123
+    },
+    {
+      "epoch": 0.0681452803569671,
+      "grad_norm": 0.8974121809005737,
+      "learning_rate": 4.8993075348363626e-05,
+      "loss": 0.8756,
+      "step": 4154
+    },
+    {
+      "epoch": 0.06865382722530267,
+      "grad_norm": 1.1337127685546875,
+      "learning_rate": 4.896934532712084e-05,
+      "loss": 0.8754,
+      "step": 4185
+    },
+    {
+      "epoch": 0.06916237409363825,
+      "grad_norm": 0.9736528396606445,
+      "learning_rate": 4.8945344815073846e-05,
+      "loss": 0.8691,
+      "step": 4216
+    },
+    {
+      "epoch": 0.06967092096197382,
+      "grad_norm": 1.0495686531066895,
+      "learning_rate": 4.892107408306516e-05,
+      "loss": 0.8845,
+      "step": 4247
+    },
+    {
+      "epoch": 0.0701794678303094,
+      "grad_norm": 1.0502818822860718,
+      "learning_rate": 4.889653340498669e-05,
+      "loss": 0.8831,
+      "step": 4278
+    },
+    {
+      "epoch": 0.07068801469864497,
+      "grad_norm": 0.9511426687240601,
+      "learning_rate": 4.8871723057776664e-05,
+      "loss": 0.8689,
+      "step": 4309
+    },
+    {
+      "epoch": 0.07119656156698054,
+      "grad_norm": 1.055638313293457,
+      "learning_rate": 4.8846643321416476e-05,
+      "loss": 0.874,
+      "step": 4340
+    },
+    {
+      "epoch": 0.07170510843531612,
+      "grad_norm": 1.098950743675232,
+      "learning_rate": 4.882129447892753e-05,
+      "loss": 0.863,
+      "step": 4371
+    },
+    {
+      "epoch": 0.07221365530365169,
+      "grad_norm": 0.8074769973754883,
+      "learning_rate": 4.8795676816368076e-05,
+      "loss": 0.8735,
+      "step": 4402
+    },
+    {
+      "epoch": 0.07272220217198727,
+      "grad_norm": 1.0250309705734253,
+      "learning_rate": 4.876979062282995e-05,
+      "loss": 0.8707,
+      "step": 4433
+    },
+    {
+      "epoch": 0.07323074904032284,
+      "grad_norm": 0.9812042117118835,
+      "learning_rate": 4.8743636190435325e-05,
+      "loss": 0.8615,
+      "step": 4464
+    },
+    {
+      "epoch": 0.07373929590865842,
+      "grad_norm": 0.8578740954399109,
+      "learning_rate": 4.871721381433344e-05,
+      "loss": 0.8748,
+      "step": 4495
+    },
+    {
+      "epoch": 0.07424784277699399,
+      "grad_norm": 0.828201413154602,
+      "learning_rate": 4.869052379269719e-05,
+      "loss": 0.8621,
+      "step": 4526
+    },
+    {
+      "epoch": 0.07475638964532957,
+      "grad_norm": 0.705862283706665,
+      "learning_rate": 4.866356642671985e-05,
+      "loss": 0.8648,
+      "step": 4557
+    },
+    {
+      "epoch": 0.07526493651366514,
+      "grad_norm": 1.1234462261199951,
+      "learning_rate": 4.8636342020611634e-05,
+      "loss": 0.8749,
+      "step": 4588
+    },
+    {
+      "epoch": 0.07577348338200073,
+      "grad_norm": 0.8310935497283936,
+      "learning_rate": 4.860885088159626e-05,
+      "loss": 0.8662,
+      "step": 4619
+    },
+    {
+      "epoch": 0.0762820302503363,
+      "grad_norm": 0.9117681980133057,
+      "learning_rate": 4.858109331990751e-05,
+      "loss": 0.8651,
+      "step": 4650
+    },
+    {
+      "epoch": 0.07679057711867188,
+      "grad_norm": 0.8817831873893738,
+      "learning_rate": 4.855306964878567e-05,
+      "loss": 0.8624,
+      "step": 4681
+    },
+    {
+      "epoch": 0.07729912398700745,
+      "grad_norm": 1.2697678804397583,
+      "learning_rate": 4.8524780184474084e-05,
+      "loss": 0.8777,
+      "step": 4712
+    },
+    {
+      "epoch": 0.07780767085534303,
+      "grad_norm": 0.9142253994941711,
+      "learning_rate": 4.8496225246215496e-05,
+      "loss": 0.863,
+      "step": 4743
+    },
+    {
+      "epoch": 0.0783162177236786,
+      "grad_norm": 0.9251907467842102,
+      "learning_rate": 4.8467405156248505e-05,
+      "loss": 0.865,
+      "step": 4774
+    },
+    {
+      "epoch": 0.07882476459201418,
+      "grad_norm": 0.8622975945472717,
+      "learning_rate": 4.843832023980392e-05,
+      "loss": 0.8642,
+      "step": 4805
+    },
+    {
+      "epoch": 0.07933331146034975,
+      "grad_norm": 0.9559304118156433,
+      "learning_rate": 4.840897082510106e-05,
+      "loss": 0.8658,
+      "step": 4836
+    },
+    {
+      "epoch": 0.07984185832868533,
+      "grad_norm": 1.0467437505722046,
+      "learning_rate": 4.8379357243344084e-05,
+      "loss": 0.8667,
+      "step": 4867
+    },
+    {
+      "epoch": 0.0803504051970209,
+      "grad_norm": 0.920857310295105,
+      "learning_rate": 4.8349479828718236e-05,
+      "loss": 0.8638,
+      "step": 4898
+    },
+    {
+      "epoch": 0.08085895206535647,
+      "grad_norm": 0.8406010866165161,
+      "learning_rate": 4.8319338918386075e-05,
+      "loss": 0.8566,
+      "step": 4929
+    },
+    {
+      "epoch": 0.08136749893369205,
+      "grad_norm": 1.535828709602356,
+      "learning_rate": 4.828893485248369e-05,
+      "loss": 0.8631,
+      "step": 4960
+    },
+    {
+      "epoch": 0.08187604580202762,
+      "grad_norm": 0.77967768907547,
+      "learning_rate": 4.825826797411682e-05,
+      "loss": 0.8664,
+      "step": 4991
+    },
+    {
+      "epoch": 0.0823845926703632,
+      "grad_norm": 0.725979745388031,
+      "learning_rate": 4.822733862935702e-05,
+      "loss": 0.865,
+      "step": 5022
+    },
+    {
+      "epoch": 0.08289313953869877,
+      "grad_norm": 0.8091695308685303,
+      "learning_rate": 4.819614716723775e-05,
+      "loss": 0.8683,
+      "step": 5053
+    },
+    {
+      "epoch": 0.08340168640703435,
+      "grad_norm": 0.8684927821159363,
+      "learning_rate": 4.8164693939750425e-05,
+      "loss": 0.8511,
+      "step": 5084
+    },
+    {
+      "epoch": 0.08391023327536992,
+      "grad_norm": 0.8640846610069275,
+      "learning_rate": 4.813297930184042e-05,
+      "loss": 0.8547,
+      "step": 5115
+    },
+    {
+      "epoch": 0.0844187801437055,
+      "grad_norm": 0.8553802371025085,
+      "learning_rate": 4.810100361140314e-05,
+      "loss": 0.8608,
+      "step": 5146
+    },
+    {
+      "epoch": 0.08492732701204107,
+      "grad_norm": 0.7264990210533142,
+      "learning_rate": 4.8068767229279885e-05,
+      "loss": 0.8586,
+      "step": 5177
+    },
+    {
+      "epoch": 0.08543587388037666,
+      "grad_norm": 0.9155231714248657,
+      "learning_rate": 4.8036270519253854e-05,
+      "loss": 0.8536,
+      "step": 5208
+    },
+    {
+      "epoch": 0.08594442074871222,
+      "grad_norm": 0.8864864706993103,
+      "learning_rate": 4.8003513848046e-05,
+      "loss": 0.856,
+      "step": 5239
+    },
+    {
+      "epoch": 0.08645296761704781,
+      "grad_norm": 0.8806331753730774,
+      "learning_rate": 4.79704975853109e-05,
+      "loss": 0.8462,
+      "step": 5270
+    },
+    {
+      "epoch": 0.08696151448538338,
+      "grad_norm": 0.8806422352790833,
+      "learning_rate": 4.793722210363262e-05,
+      "loss": 0.8548,
+      "step": 5301
+    },
+    {
+      "epoch": 0.08747006135371896,
+      "grad_norm": 0.9075062870979309,
+      "learning_rate": 4.7903687778520414e-05,
+      "loss": 0.86,
+      "step": 5332
+    },
+    {
+      "epoch": 0.08797860822205453,
+      "grad_norm": 0.952202320098877,
+      "learning_rate": 4.7869894988404593e-05,
+      "loss": 0.8523,
+      "step": 5363
+    },
+    {
+      "epoch": 0.08848715509039011,
+      "grad_norm": 0.7852069139480591,
+      "learning_rate": 4.783584411463221e-05,
+      "loss": 0.8573,
+      "step": 5394
+    },
+    {
+      "epoch": 0.08899570195872568,
+      "grad_norm": 0.7687172889709473,
+      "learning_rate": 4.780153554146274e-05,
+      "loss": 0.846,
+      "step": 5425
+    },
+    {
+      "epoch": 0.08950424882706126,
+      "grad_norm": 0.8335172533988953,
+      "learning_rate": 4.7766969656063766e-05,
+      "loss": 0.8532,
+      "step": 5456
+    },
+    {
+      "epoch": 0.09001279569539683,
+      "grad_norm": 0.8502724170684814,
+      "learning_rate": 4.773214684850662e-05,
+      "loss": 0.8461,
+      "step": 5487
+    },
+    {
+      "epoch": 0.09052134256373241,
+      "grad_norm": 0.9617708325386047,
+      "learning_rate": 4.769706751176193e-05,
+      "loss": 0.8501,
+      "step": 5518
+    },
+    {
+      "epoch": 0.09102988943206798,
+      "grad_norm": 0.843956470489502,
+      "learning_rate": 4.7661732041695264e-05,
+      "loss": 0.8498,
+      "step": 5549
+    },
+    {
+      "epoch": 0.09153843630040355,
+      "grad_norm": 0.7345194220542908,
+      "learning_rate": 4.762614083706258e-05,
+      "loss": 0.8595,
+      "step": 5580
+    },
+    {
+      "epoch": 0.09204698316873913,
+      "grad_norm": 0.8171796202659607,
+      "learning_rate": 4.759029429950581e-05,
+      "loss": 0.8499,
+      "step": 5611
+    },
+    {
+      "epoch": 0.0925555300370747,
+      "grad_norm": 0.9034848213195801,
+      "learning_rate": 4.7554192833548235e-05,
+      "loss": 0.8515,
+      "step": 5642
+    },
+    {
+      "epoch": 0.09306407690541028,
+      "grad_norm": 0.8186687231063843,
+      "learning_rate": 4.751783684659e-05,
+      "loss": 0.852,
+      "step": 5673
+    },
+    {
+      "epoch": 0.09357262377374585,
+      "grad_norm": 0.8931159377098083,
+      "learning_rate": 4.748122674890348e-05,
+      "loss": 0.8471,
+      "step": 5704
+    },
+    {
+      "epoch": 0.09408117064208144,
+      "grad_norm": 0.9406206011772156,
+      "learning_rate": 4.7444362953628654e-05,
+      "loss": 0.8494,
+      "step": 5735
+    },
+    {
+      "epoch": 0.094589717510417,
+      "grad_norm": 0.7873674035072327,
+      "learning_rate": 4.7407245876768424e-05,
+      "loss": 0.8498,
+      "step": 5766
+    },
+    {
+      "epoch": 0.09509826437875259,
+      "grad_norm": 0.8357398509979248,
+      "learning_rate": 4.736987593718397e-05,
+      "loss": 0.8539,
+      "step": 5797
+    },
+    {
+      "epoch": 0.09560681124708816,
+      "grad_norm": 0.9035603404045105,
+      "learning_rate": 4.733225355658999e-05,
+      "loss": 0.8492,
+      "step": 5828
+    },
+    {
+      "epoch": 0.09611535811542374,
+      "grad_norm": 0.9840619564056396,
+      "learning_rate": 4.7294379159549926e-05,
+      "loss": 0.8556,
+      "step": 5859
+    },
+    {
+      "epoch": 0.0966239049837593,
+      "grad_norm": 0.9834292531013489,
+      "learning_rate": 4.725625317347119e-05,
+      "loss": 0.8419,
+      "step": 5890
+    },
+    {
+      "epoch": 0.09713245185209489,
+      "grad_norm": 0.9010293483734131,
+      "learning_rate": 4.7217876028600374e-05,
+      "loss": 0.8536,
+      "step": 5921
+    },
+    {
+      "epoch": 0.09764099872043046,
+      "grad_norm": 0.8637653589248657,
+      "learning_rate": 4.717924815801832e-05,
+      "loss": 0.8557,
+      "step": 5952
+    },
+    {
+      "epoch": 0.09814954558876604,
+      "grad_norm": 0.8036808967590332,
+      "learning_rate": 4.714036999763532e-05,
+      "loss": 0.8379,
+      "step": 5983
+    },
+    {
+      "epoch": 0.09865809245710161,
+      "grad_norm": 0.70550936460495,
+      "learning_rate": 4.7101241986186116e-05,
+      "loss": 0.8514,
+      "step": 6014
+    },
+    {
+      "epoch": 0.09916663932543719,
+      "grad_norm": 0.8050324320793152,
+      "learning_rate": 4.7061864565225e-05,
+      "loss": 0.8561,
+      "step": 6045
+    },
+    {
+      "epoch": 0.09967518619377276,
+      "grad_norm": 0.8765999674797058,
+      "learning_rate": 4.702223817912081e-05,
+      "loss": 0.842,
+      "step": 6076
+    },
+    {
+      "epoch": 0.10018373306210834,
+      "grad_norm": 0.8884183764457703,
+      "learning_rate": 4.698236327505195e-05,
+      "loss": 0.842,
+      "step": 6107
+    },
+    {
+      "epoch": 0.10069227993044391,
+      "grad_norm": 0.7993413209915161,
+      "learning_rate": 4.694224030300127e-05,
+      "loss": 0.8346,
+      "step": 6138
+    },
+    {
+      "epoch": 0.10120082679877948,
+      "grad_norm": 0.8383135199546814,
+      "learning_rate": 4.690186971575107e-05,
+      "loss": 0.84,
+      "step": 6169
+    },
+    {
+      "epoch": 0.10170937366711506,
+      "grad_norm": 0.9864751100540161,
+      "learning_rate": 4.6861251968877916e-05,
+      "loss": 0.8475,
+      "step": 6200
+    },
+    {
+      "epoch": 0.10221792053545063,
+      "grad_norm": 0.7869567275047302,
+      "learning_rate": 4.68203875207476e-05,
+      "loss": 0.8541,
+      "step": 6231
+    },
+    {
+      "epoch": 0.10272646740378621,
+      "grad_norm": 0.8401315212249756,
+      "learning_rate": 4.677927683250983e-05,
+      "loss": 0.8498,
+      "step": 6262
+    },
+    {
+      "epoch": 0.10323501427212178,
+      "grad_norm": 0.8147920370101929,
+      "learning_rate": 4.6737920368093156e-05,
+      "loss": 0.8521,
+      "step": 6293
+    },
+    {
+      "epoch": 0.10374356114045737,
+      "grad_norm": 0.9189819097518921,
+      "learning_rate": 4.669631859419965e-05,
+      "loss": 0.8358,
+      "step": 6324
+    },
+    {
+      "epoch": 0.10425210800879293,
+      "grad_norm": 0.8106151819229126,
+      "learning_rate": 4.6654471980299676e-05,
+      "loss": 0.8355,
+      "step": 6355
+    },
+    {
+      "epoch": 0.10476065487712852,
+      "grad_norm": 0.8246133327484131,
+      "learning_rate": 4.661238099862658e-05,
+      "loss": 0.8502,
+      "step": 6386
+    },
+    {
+      "epoch": 0.10526920174546409,
+      "grad_norm": 0.7696080803871155,
+      "learning_rate": 4.657004612417138e-05,
+      "loss": 0.8452,
+      "step": 6417
+    },
+    {
+      "epoch": 0.10577774861379967,
+      "grad_norm": 0.8194378018379211,
+      "learning_rate": 4.6527467834677374e-05,
+      "loss": 0.8538,
+      "step": 6448
+    },
+    {
+      "epoch": 0.10628629548213524,
+      "grad_norm": 0.763508677482605,
+      "learning_rate": 4.648464661063478e-05,
+      "loss": 0.8394,
+      "step": 6479
+    },
+    {
+      "epoch": 0.10679484235047082,
+      "grad_norm": 0.8376573920249939,
+      "learning_rate": 4.6441582935275264e-05,
+      "loss": 0.8449,
+      "step": 6510
+    },
+    {
+      "epoch": 0.10730338921880639,
+      "grad_norm": 0.7334773540496826,
+      "learning_rate": 4.6398277294566586e-05,
+      "loss": 0.8458,
+      "step": 6541
+    },
+    {
+      "epoch": 0.10781193608714197,
+      "grad_norm": 0.8620659112930298,
+      "learning_rate": 4.6354730177207e-05,
+      "loss": 0.8318,
+      "step": 6572
+    },
+    {
+      "epoch": 0.10832048295547754,
+      "grad_norm": 0.8515760898590088,
+      "learning_rate": 4.6310942074619787e-05,
+      "loss": 0.8424,
+      "step": 6603
+    },
+    {
+      "epoch": 0.10882902982381312,
+      "grad_norm": 0.7571322321891785,
+      "learning_rate": 4.626691348094777e-05,
+      "loss": 0.8354,
+      "step": 6634
+    },
+    {
+      "epoch": 0.10933757669214869,
+      "grad_norm": 0.7566030025482178,
+      "learning_rate": 4.622264489304762e-05,
+      "loss": 0.8358,
+      "step": 6665
+    },
+    {
+      "epoch": 0.10984612356048427,
+      "grad_norm": 0.9415192604064941,
+      "learning_rate": 4.617813681048434e-05,
+      "loss": 0.8407,
+      "step": 6696
+    },
+    {
+      "epoch": 0.11035467042881984,
+      "grad_norm": 0.9476765990257263,
+      "learning_rate": 4.61333897355256e-05,
+      "loss": 0.8348,
+      "step": 6727
+    },
+    {
+      "epoch": 0.11086321729715543,
+      "grad_norm": 0.7231162190437317,
+      "learning_rate": 4.608840417313604e-05,
+      "loss": 0.8287,
+      "step": 6758
+    },
+    {
+      "epoch": 0.111371764165491,
+      "grad_norm": 0.8302910327911377,
+      "learning_rate": 4.6043180630971646e-05,
+      "loss": 0.8416,
+      "step": 6789
+    },
+    {
+      "epoch": 0.11188031103382656,
+      "grad_norm": 0.7198696732521057,
+      "learning_rate": 4.599771961937391e-05,
+      "loss": 0.8483,
+      "step": 6820
+    },
+    {
+      "epoch": 0.11238885790216215,
+      "grad_norm": 0.843404233455658,
+      "learning_rate": 4.5952021651364204e-05,
+      "loss": 0.8326,
+      "step": 6851
+    },
+    {
+      "epoch": 0.11289740477049771,
+      "grad_norm": 0.7174884080886841,
+      "learning_rate": 4.590608724263786e-05,
+      "loss": 0.8389,
+      "step": 6882
+    },
+    {
+      "epoch": 0.1134059516388333,
+      "grad_norm": 0.9392669796943665,
+      "learning_rate": 4.585991691155845e-05,
+      "loss": 0.8373,
+      "step": 6913
+    },
+    {
+      "epoch": 0.11391449850716887,
+      "grad_norm": 0.7740927934646606,
+      "learning_rate": 4.581351117915188e-05,
+      "loss": 0.8269,
+      "step": 6944
+    },
+    {
+      "epoch": 0.11442304537550445,
+      "grad_norm": 0.7788698673248291,
+      "learning_rate": 4.5766870569100534e-05,
+      "loss": 0.8414,
+      "step": 6975
+    },
+    {
+      "epoch": 0.11493159224384002,
+      "grad_norm": 0.9135116934776306,
+      "learning_rate": 4.571999560773736e-05,
+      "loss": 0.8348,
+      "step": 7006
+    },
+    {
+      "epoch": 0.1154401391121756,
+      "grad_norm": 0.7956461906433105,
+      "learning_rate": 4.5672886824039915e-05,
+      "loss": 0.8383,
+      "step": 7037
+    },
+    {
+      "epoch": 0.11594868598051117,
+      "grad_norm": 0.8202860355377197,
+      "learning_rate": 4.5625544749624435e-05,
+      "loss": 0.8351,
+      "step": 7068
+    },
+    {
+      "epoch": 0.11645723284884675,
+      "grad_norm": 0.7899375557899475,
+      "learning_rate": 4.5577969918739794e-05,
+      "loss": 0.835,
+      "step": 7099
+    },
+    {
+      "epoch": 0.11696577971718232,
+      "grad_norm": 0.7905834913253784,
+      "learning_rate": 4.5530162868261486e-05,
+      "loss": 0.8321,
+      "step": 7130
+    },
+    {
+      "epoch": 0.1174743265855179,
+      "grad_norm": 0.7519370913505554,
+      "learning_rate": 4.548212413768558e-05,
+      "loss": 0.8333,
+      "step": 7161
+    },
+    {
+      "epoch": 0.11798287345385347,
+      "grad_norm": 0.8854320645332336,
+      "learning_rate": 4.543385426912261e-05,
+      "loss": 0.8261,
+      "step": 7192
+    },
+    {
+      "epoch": 0.11849142032218905,
+      "grad_norm": 0.8104952573776245,
+      "learning_rate": 4.53853538072915e-05,
+      "loss": 0.8378,
+      "step": 7223
+    },
+    {
+      "epoch": 0.11899996719052462,
+      "grad_norm": 0.8440843820571899,
+      "learning_rate": 4.533662329951336e-05,
+      "loss": 0.8363,
+      "step": 7254
+    },
+    {
+      "epoch": 0.1195085140588602,
+      "grad_norm": 0.8464990854263306,
+      "learning_rate": 4.528766329570536e-05,
+      "loss": 0.8472,
+      "step": 7285
+    },
+    {
+      "epoch": 0.12001706092719577,
+      "grad_norm": 0.7120764851570129,
+      "learning_rate": 4.523847434837447e-05,
+      "loss": 0.8439,
+      "step": 7316
+    },
+    {
+      "epoch": 0.12052560779553136,
+      "grad_norm": 0.8461644053459167,
+      "learning_rate": 4.518905701261128e-05,
+      "loss": 0.8359,
+      "step": 7347
+    },
+    {
+      "epoch": 0.12103415466386692,
+      "grad_norm": 0.7484918236732483,
+      "learning_rate": 4.5139411846083715e-05,
+      "loss": 0.8362,
+      "step": 7378
+    },
+    {
+      "epoch": 0.1215427015322025,
+      "grad_norm": 0.759344756603241,
+      "learning_rate": 4.508953940903073e-05,
+      "loss": 0.8402,
+      "step": 7409
+    },
+    {
+      "epoch": 0.12205124840053808,
+      "grad_norm": 0.7918707728385925,
+      "learning_rate": 4.5039440264255994e-05,
+      "loss": 0.8381,
+      "step": 7440
+    },
+    {
+      "epoch": 0.12255979526887364,
+      "grad_norm": 0.7695263624191284,
+      "learning_rate": 4.498911497712155e-05,
+      "loss": 0.8362,
+      "step": 7471
+    },
+    {
+      "epoch": 0.12306834213720923,
+      "grad_norm": 0.7486735582351685,
+      "learning_rate": 4.493856411554142e-05,
+      "loss": 0.8282,
+      "step": 7502
+    },
+    {
+      "epoch": 0.1235768890055448,
+      "grad_norm": 0.9669325947761536,
+      "learning_rate": 4.4887788249975206e-05,
+      "loss": 0.8219,
+      "step": 7533
+    },
+    {
+      "epoch": 0.12408543587388038,
+      "grad_norm": 0.7665596604347229,
+      "learning_rate": 4.4836787953421656e-05,
+      "loss": 0.8316,
+      "step": 7564
+    },
+    {
+      "epoch": 0.12459398274221595,
+      "grad_norm": 0.6568590402603149,
+      "learning_rate": 4.478556380141218e-05,
+      "loss": 0.8309,
+      "step": 7595
+    },
+    {
+      "epoch": 0.12510252961055152,
+      "grad_norm": 0.7445078492164612,
+      "learning_rate": 4.4734116372004375e-05,
+      "loss": 0.8375,
+      "step": 7626
+    },
+    {
+      "epoch": 0.1256110764788871,
+      "grad_norm": 0.7931857109069824,
+      "learning_rate": 4.4682446245775477e-05,
+      "loss": 0.8299,
+      "step": 7657
+    },
+    {
+      "epoch": 0.12611962334722268,
+      "grad_norm": 0.7949565052986145,
+      "learning_rate": 4.463055400581586e-05,
+      "loss": 0.824,
+      "step": 7688
+    },
+    {
+      "epoch": 0.12662817021555825,
+      "grad_norm": 0.9419664144515991,
+      "learning_rate": 4.4578440237722374e-05,
+      "loss": 0.8279,
+      "step": 7719
+    },
+    {
+      "epoch": 0.12713671708389382,
+      "grad_norm": 0.8755522966384888,
+      "learning_rate": 4.452610552959183e-05,
+      "loss": 0.8323,
+      "step": 7750
+    },
+    {
+      "epoch": 0.12764526395222942,
+      "grad_norm": 0.7803270816802979,
+      "learning_rate": 4.447355047201428e-05,
+      "loss": 0.8288,
+      "step": 7781
+    },
+    {
+      "epoch": 0.12815381082056498,
+      "grad_norm": 0.8153209686279297,
+      "learning_rate": 4.4420775658066414e-05,
+      "loss": 0.8311,
+      "step": 7812
+    },
+    {
+      "epoch": 0.12866235768890055,
+      "grad_norm": 0.7577016949653625,
+      "learning_rate": 4.436778168330484e-05,
+      "loss": 0.8263,
+      "step": 7843
+    },
+    {
+      "epoch": 0.12917090455723612,
+      "grad_norm": 0.7444413304328918,
+      "learning_rate": 4.4314569145759353e-05,
+      "loss": 0.8248,
+      "step": 7874
+    },
+    {
+      "epoch": 0.12967945142557172,
+      "grad_norm": 0.7085497379302979,
+      "learning_rate": 4.42611386459262e-05,
+      "loss": 0.8299,
+      "step": 7905
+    },
+    {
+      "epoch": 0.1301879982939073,
+      "grad_norm": 0.9205734133720398,
+      "learning_rate": 4.420749078676133e-05,
+      "loss": 0.8361,
+      "step": 7936
+    },
+    {
+      "epoch": 0.13069654516224286,
+      "grad_norm": 0.7982526421546936,
+      "learning_rate": 4.4153626173673516e-05,
+      "loss": 0.8285,
+      "step": 7967
+    },
+    {
+      "epoch": 0.13120509203057842,
+      "grad_norm": 0.7862482070922852,
+      "learning_rate": 4.409954541451762e-05,
+      "loss": 0.8295,
+      "step": 7998
+    },
+    {
+      "epoch": 0.131713638898914,
+      "grad_norm": 0.6587911248207092,
+      "learning_rate": 4.404524911958764e-05,
+      "loss": 0.8352,
+      "step": 8029
+    },
+    {
+      "epoch": 0.1322221857672496,
+      "grad_norm": 0.9410633444786072,
+      "learning_rate": 4.399073790160989e-05,
+      "loss": 0.8309,
+      "step": 8060
+    },
+    {
+      "epoch": 0.13273073263558516,
+      "grad_norm": 0.7766574025154114,
+      "learning_rate": 4.393601237573607e-05,
+      "loss": 0.8246,
+      "step": 8091
+    },
+    {
+      "epoch": 0.13323927950392073,
+      "grad_norm": 0.7887439131736755,
+      "learning_rate": 4.388107315953628e-05,
+      "loss": 0.8288,
+      "step": 8122
+    },
+    {
+      "epoch": 0.1337478263722563,
+      "grad_norm": 0.9374856352806091,
+      "learning_rate": 4.382592087299212e-05,
+      "loss": 0.8306,
+      "step": 8153
+    },
+    {
+      "epoch": 0.1342563732405919,
+      "grad_norm": 0.9233435988426208,
+      "learning_rate": 4.377055613848964e-05,
+      "loss": 0.8268,
+      "step": 8184
+    },
+    {
+      "epoch": 0.13476492010892746,
+      "grad_norm": 0.8031365871429443,
+      "learning_rate": 4.3714979580812355e-05,
+      "loss": 0.8245,
+      "step": 8215
+    },
+    {
+      "epoch": 0.13527346697726303,
+      "grad_norm": 0.654653787612915,
+      "learning_rate": 4.365919182713416e-05,
+      "loss": 0.8255,
+      "step": 8246
+    },
+    {
+      "epoch": 0.1357820138455986,
+      "grad_norm": 0.7165628671646118,
+      "learning_rate": 4.360319350701226e-05,
+      "loss": 0.8322,
+      "step": 8277
+    },
+    {
+      "epoch": 0.1362905607139342,
+      "grad_norm": 0.7504312992095947,
+      "learning_rate": 4.3546985252380115e-05,
+      "loss": 0.8229,
+      "step": 8308
+    },
+    {
+      "epoch": 0.13679910758226976,
+      "grad_norm": 0.9588577747344971,
+      "learning_rate": 4.349056769754021e-05,
+      "loss": 0.8289,
+      "step": 8339
+    },
+    {
+      "epoch": 0.13730765445060533,
+      "grad_norm": 1.0719257593154907,
+      "learning_rate": 4.3433941479156994e-05,
+      "loss": 0.8262,
+      "step": 8370
+    },
+    {
+      "epoch": 0.1378162013189409,
+      "grad_norm": 0.7595779299736023,
+      "learning_rate": 4.3377107236249647e-05,
+      "loss": 0.8274,
+      "step": 8401
+    },
+    {
+      "epoch": 0.1383247481872765,
+      "grad_norm": 0.7770616412162781,
+      "learning_rate": 4.332006561018488e-05,
+      "loss": 0.8253,
+      "step": 8432
+    },
+    {
+      "epoch": 0.13883329505561207,
+      "grad_norm": 0.8065539598464966,
+      "learning_rate": 4.3262817244669683e-05,
+      "loss": 0.8333,
+      "step": 8463
+    },
+    {
+      "epoch": 0.13934184192394763,
+      "grad_norm": 0.8903069496154785,
+      "learning_rate": 4.3205362785744083e-05,
+      "loss": 0.8203,
+      "step": 8494
+    },
+    {
+      "epoch": 0.1398503887922832,
+      "grad_norm": 0.8108446598052979,
+      "learning_rate": 4.314770288177384e-05,
+      "loss": 0.82,
+      "step": 8525
+    },
+    {
+      "epoch": 0.1403589356606188,
+      "grad_norm": 0.8944516181945801,
+      "learning_rate": 4.308983818344313e-05,
+      "loss": 0.821,
+      "step": 8556
+    },
+    {
+      "epoch": 0.14086748252895437,
+      "grad_norm": 0.7374255061149597,
+      "learning_rate": 4.3031769343747206e-05,
+      "loss": 0.8145,
+      "step": 8587
+    },
+    {
+      "epoch": 0.14137602939728994,
+      "grad_norm": 0.7687476873397827,
+      "learning_rate": 4.297349701798505e-05,
+      "loss": 0.8158,
+      "step": 8618
+    },
+    {
+      "epoch": 0.1418845762656255,
+      "grad_norm": 0.8074747323989868,
+      "learning_rate": 4.2915021863751916e-05,
+      "loss": 0.8161,
+      "step": 8649
+    },
+    {
+      "epoch": 0.14239312313396107,
+      "grad_norm": 0.6590523719787598,
+      "learning_rate": 4.285634454093198e-05,
+      "loss": 0.8205,
+      "step": 8680
+    },
+    {
+      "epoch": 0.14290167000229667,
+      "grad_norm": 0.7120646834373474,
+      "learning_rate": 4.279746571169086e-05,
+      "loss": 0.8211,
+      "step": 8711
+    },
+    {
+      "epoch": 0.14341021687063224,
+      "grad_norm": 0.6680018305778503,
+      "learning_rate": 4.2738386040468136e-05,
+      "loss": 0.8262,
+      "step": 8742
+    },
+    {
+      "epoch": 0.1439187637389678,
+      "grad_norm": 0.6493867039680481,
+      "learning_rate": 4.2679106193969866e-05,
+      "loss": 0.8138,
+      "step": 8773
+    },
+    {
+      "epoch": 0.14442731060730338,
+      "grad_norm": 0.8403043746948242,
+      "learning_rate": 4.261962684116106e-05,
+      "loss": 0.8252,
+      "step": 8804
+    },
+    {
+      "epoch": 0.14493585747563897,
+      "grad_norm": 0.8730862140655518,
+      "learning_rate": 4.2559948653258145e-05,
+      "loss": 0.8207,
+      "step": 8835
+    },
+    {
+      "epoch": 0.14544440434397454,
+      "grad_norm": 0.7768038511276245,
+      "learning_rate": 4.250007230372134e-05,
+      "loss": 0.8301,
+      "step": 8866
+    },
+    {
+      "epoch": 0.1459529512123101,
+      "grad_norm": 0.7834228277206421,
+      "learning_rate": 4.2439998468247126e-05,
+      "loss": 0.8279,
+      "step": 8897
+    },
+    {
+      "epoch": 0.14646149808064568,
+      "grad_norm": 0.7046851515769958,
+      "learning_rate": 4.2379727824760566e-05,
+      "loss": 0.8191,
+      "step": 8928
+    },
+    {
+      "epoch": 0.14697004494898128,
+      "grad_norm": 0.7662693858146667,
+      "learning_rate": 4.231926105340768e-05,
+      "loss": 0.8265,
+      "step": 8959
+    },
+    {
+      "epoch": 0.14747859181731685,
+      "grad_norm": 0.6945628523826599,
+      "learning_rate": 4.225859883654776e-05,
+      "loss": 0.8164,
+      "step": 8990
+    },
+    {
+      "epoch": 0.14798713868565241,
+      "grad_norm": 0.7956746220588684,
+      "learning_rate": 4.219774185874569e-05,
+      "loss": 0.8249,
+      "step": 9021
+    },
+    {
+      "epoch": 0.14849568555398798,
+      "grad_norm": 0.6459651589393616,
+      "learning_rate": 4.213669080676418e-05,
+      "loss": 0.8118,
+      "step": 9052
+    },
+    {
+      "epoch": 0.14900423242232358,
+      "grad_norm": 0.6358417272567749,
+      "learning_rate": 4.2075446369556056e-05,
+      "loss": 0.8177,
+      "step": 9083
+    },
+    {
+      "epoch": 0.14951277929065915,
+      "grad_norm": 0.7192367911338806,
+      "learning_rate": 4.201400923825648e-05,
+      "loss": 0.8164,
+      "step": 9114
+    },
+    {
+      "epoch": 0.15002132615899472,
+      "grad_norm": 0.7007993459701538,
+      "learning_rate": 4.195238010617511e-05,
+      "loss": 0.8182,
+      "step": 9145
+    },
+    {
+      "epoch": 0.15052987302733029,
+      "grad_norm": 0.8889499306678772,
+      "learning_rate": 4.1890559668788344e-05,
+      "loss": 0.8239,
+      "step": 9176
+    },
+    {
+      "epoch": 0.15103841989566588,
+      "grad_norm": 0.7275688648223877,
+      "learning_rate": 4.1828548623731405e-05,
+      "loss": 0.8272,
+      "step": 9207
+    },
+    {
+      "epoch": 0.15154696676400145,
+      "grad_norm": 0.8763214349746704,
+      "learning_rate": 4.1766347670790506e-05,
+      "loss": 0.8133,
+      "step": 9238
+    },
+    {
+      "epoch": 0.15205551363233702,
+      "grad_norm": 0.7568575143814087,
+      "learning_rate": 4.170395751189495e-05,
+      "loss": 0.8301,
+      "step": 9269
+    },
+    {
+      "epoch": 0.1525640605006726,
+      "grad_norm": 0.8635027408599854,
+      "learning_rate": 4.164137885110921e-05,
+      "loss": 0.8191,
+      "step": 9300
+    },
+    {
+      "epoch": 0.15307260736900816,
+      "grad_norm": 1.1189345121383667,
+      "learning_rate": 4.157861239462495e-05,
+      "loss": 0.8251,
+      "step": 9331
+    },
+    {
+      "epoch": 0.15358115423734375,
+      "grad_norm": 0.7589987516403198,
+      "learning_rate": 4.1515658850753114e-05,
+      "loss": 0.8173,
+      "step": 9362
+    },
+    {
+      "epoch": 0.15408970110567932,
+      "grad_norm": 0.8124852180480957,
+      "learning_rate": 4.145251892991588e-05,
+      "loss": 0.8245,
+      "step": 9393
+    },
+    {
+      "epoch": 0.1545982479740149,
+      "grad_norm": 0.8820806741714478,
+      "learning_rate": 4.138919334463868e-05,
+      "loss": 0.8142,
+      "step": 9424
+    },
+    {
+      "epoch": 0.15510679484235046,
+      "grad_norm": 0.6851283311843872,
+      "learning_rate": 4.1325682809542124e-05,
+      "loss": 0.7997,
+      "step": 9455
+    },
+    {
+      "epoch": 0.15561534171068606,
+      "grad_norm": 0.8576741814613342,
+      "learning_rate": 4.126198804133398e-05,
+      "loss": 0.8188,
+      "step": 9486
+    },
+    {
+      "epoch": 0.15612388857902162,
+      "grad_norm": 0.7869651317596436,
+      "learning_rate": 4.1198109758801055e-05,
+      "loss": 0.8218,
+      "step": 9517
+    },
+    {
+      "epoch": 0.1566324354473572,
+      "grad_norm": 0.7563472390174866,
+      "learning_rate": 4.113404868280107e-05,
+      "loss": 0.8245,
+      "step": 9548
+    },
+    {
+      "epoch": 0.15714098231569276,
+      "grad_norm": 0.7451006174087524,
+      "learning_rate": 4.106980553625457e-05,
+      "loss": 0.8072,
+      "step": 9579
+    },
+    {
+      "epoch": 0.15764952918402836,
+      "grad_norm": 0.6817291378974915,
+      "learning_rate": 4.100538104413674e-05,
+      "loss": 0.8175,
+      "step": 9610
+    },
+    {
+      "epoch": 0.15815807605236393,
+      "grad_norm": 0.7386301755905151,
+      "learning_rate": 4.09407759334692e-05,
+      "loss": 0.8147,
+      "step": 9641
+    },
+    {
+      "epoch": 0.1586666229206995,
+      "grad_norm": 0.7530351877212524,
+      "learning_rate": 4.087599093331186e-05,
+      "loss": 0.8246,
+      "step": 9672
+    },
+    {
+      "epoch": 0.15917516978903506,
+      "grad_norm": 0.7016597390174866,
+      "learning_rate": 4.081102677475462e-05,
+      "loss": 0.809,
+      "step": 9703
+    },
+    {
+      "epoch": 0.15968371665737066,
+      "grad_norm": 0.7503277063369751,
+      "learning_rate": 4.0745884190909194e-05,
+      "loss": 0.8203,
+      "step": 9734
+    },
+    {
+      "epoch": 0.16019226352570623,
+      "grad_norm": 0.8983309268951416,
+      "learning_rate": 4.0680563916900796e-05,
+      "loss": 0.8265,
+      "step": 9765
+    },
+    {
+      "epoch": 0.1607008103940418,
+      "grad_norm": 0.8664911985397339,
+      "learning_rate": 4.0615066689859815e-05,
+      "loss": 0.8096,
+      "step": 9796
+    },
+    {
+      "epoch": 0.16120935726237737,
+      "grad_norm": 0.882965087890625,
+      "learning_rate": 4.0549393248913584e-05,
+      "loss": 0.8131,
+      "step": 9827
+    },
+    {
+      "epoch": 0.16171790413071294,
+      "grad_norm": 0.7006222605705261,
+      "learning_rate": 4.048354433517794e-05,
+      "loss": 0.8184,
+      "step": 9858
+    },
+    {
+      "epoch": 0.16222645099904853,
+      "grad_norm": 0.749029815196991,
+      "learning_rate": 4.0417520691748916e-05,
+      "loss": 0.8227,
+      "step": 9889
+    },
+    {
+      "epoch": 0.1627349978673841,
+      "grad_norm": 0.7951566576957703,
+      "learning_rate": 4.035132306369438e-05,
+      "loss": 0.8152,
+      "step": 9920
+    },
+    {
+      "epoch": 0.16324354473571967,
+      "grad_norm": 0.8387523293495178,
+      "learning_rate": 4.028495219804555e-05,
+      "loss": 0.8148,
+      "step": 9951
+    },
+    {
+      "epoch": 0.16375209160405524,
+      "grad_norm": 0.7745356559753418,
+      "learning_rate": 4.021840884378864e-05,
+      "loss": 0.8068,
+      "step": 9982
+    },
+    {
+      "epoch": 0.16426063847239084,
+      "grad_norm": 0.8473168611526489,
+      "learning_rate": 4.015169375185633e-05,
+      "loss": 0.8084,
+      "step": 10013
+    },
+    {
+      "epoch": 0.1647691853407264,
+      "grad_norm": 0.8187867403030396,
+      "learning_rate": 4.0084807675119396e-05,
+      "loss": 0.8051,
+      "step": 10044
+    },
+    {
+      "epoch": 0.16527773220906197,
+      "grad_norm": 0.8002802133560181,
+      "learning_rate": 4.0017751368378106e-05,
+      "loss": 0.817,
+      "step": 10075
+    },
+    {
+      "epoch": 0.16578627907739754,
+      "grad_norm": 0.6238591074943542,
+      "learning_rate": 3.995052558835377e-05,
+      "loss": 0.8076,
+      "step": 10106
+    },
+    {
+      "epoch": 0.16629482594573314,
+      "grad_norm": 0.7085210680961609,
+      "learning_rate": 3.988313109368017e-05,
+      "loss": 0.8115,
+      "step": 10137
+    },
+    {
+      "epoch": 0.1668033728140687,
+      "grad_norm": 0.6768177151679993,
+      "learning_rate": 3.981556864489504e-05,
+      "loss": 0.8112,
+      "step": 10168
+    },
+    {
+      "epoch": 0.16731191968240428,
+      "grad_norm": 0.7513672113418579,
+      "learning_rate": 3.974783900443142e-05,
+      "loss": 0.8065,
+      "step": 10199
+    },
+    {
+      "epoch": 0.16782046655073984,
+      "grad_norm": 0.7792861461639404,
+      "learning_rate": 3.9679942936609095e-05,
+      "loss": 0.8116,
+      "step": 10230
+    },
+    {
+      "epoch": 0.16832901341907544,
+      "grad_norm": 0.7917279005050659,
+      "learning_rate": 3.961188120762596e-05,
+      "loss": 0.8167,
+      "step": 10261
+    },
+    {
+      "epoch": 0.168837560287411,
+      "grad_norm": 0.6312729120254517,
+      "learning_rate": 3.954365458554938e-05,
+      "loss": 0.8001,
+      "step": 10292
+    },
+    {
+      "epoch": 0.16934610715574658,
+      "grad_norm": 0.7990860342979431,
+      "learning_rate": 3.947526384030751e-05,
+      "loss": 0.8144,
+      "step": 10323
+    },
+    {
+      "epoch": 0.16985465402408215,
+      "grad_norm": 0.6678237915039062,
+      "learning_rate": 3.9406709743680624e-05,
+      "loss": 0.8188,
+      "step": 10354
+    },
+    {
+      "epoch": 0.17036320089241774,
+      "grad_norm": 0.7347424030303955,
+      "learning_rate": 3.9337993069292366e-05,
+      "loss": 0.8046,
+      "step": 10385
+    },
+    {
+      "epoch": 0.1708717477607533,
+      "grad_norm": 0.8631488084793091,
+      "learning_rate": 3.926911459260109e-05,
+      "loss": 0.8083,
+      "step": 10416
+    },
+    {
+      "epoch": 0.17138029462908888,
+      "grad_norm": 0.9645228981971741,
+      "learning_rate": 3.920007509089102e-05,
+      "loss": 0.81,
+      "step": 10447
+    },
+    {
+      "epoch": 0.17188884149742445,
+      "grad_norm": 0.6585387587547302,
+      "learning_rate": 3.913087534326357e-05,
+      "loss": 0.8044,
+      "step": 10478
+    },
+    {
+      "epoch": 0.17239738836576002,
+      "grad_norm": 0.73898845911026,
+      "learning_rate": 3.9061516130628475e-05,
+      "loss": 0.8042,
+      "step": 10509
+    },
+    {
+      "epoch": 0.17290593523409561,
+      "grad_norm": 0.6710891127586365,
+      "learning_rate": 3.8991998235695025e-05,
+      "loss": 0.8083,
+      "step": 10540
+    },
+    {
+      "epoch": 0.17341448210243118,
+      "grad_norm": 0.9093446731567383,
+      "learning_rate": 3.8922322442963224e-05,
+      "loss": 0.8138,
+      "step": 10571
+    },
+    {
+      "epoch": 0.17392302897076675,
+      "grad_norm": 0.9384357929229736,
+      "learning_rate": 3.885248953871491e-05,
+      "loss": 0.8164,
+      "step": 10602
+    },
+    {
+      "epoch": 0.17443157583910232,
+      "grad_norm": 0.7147036790847778,
+      "learning_rate": 3.8782500311004915e-05,
+      "loss": 0.8098,
+      "step": 10633
+    },
+    {
+      "epoch": 0.17494012270743792,
+      "grad_norm": 0.637306809425354,
+      "learning_rate": 3.871235554965218e-05,
+      "loss": 0.8073,
+      "step": 10664
+    },
+    {
+      "epoch": 0.17544866957577349,
+      "grad_norm": 0.8120321035385132,
+      "learning_rate": 3.864205604623078e-05,
+      "loss": 0.8025,
+      "step": 10695
+    },
+    {
+      "epoch": 0.17595721644410905,
+      "grad_norm": 0.6488457322120667,
+      "learning_rate": 3.857160259406107e-05,
+      "loss": 0.8064,
+      "step": 10726
+    },
+    {
+      "epoch": 0.17646576331244462,
+      "grad_norm": 0.7131192684173584,
+      "learning_rate": 3.8500995988200674e-05,
+      "loss": 0.7985,
+      "step": 10757
+    },
+    {
+      "epoch": 0.17697431018078022,
+      "grad_norm": 0.670443594455719,
+      "learning_rate": 3.843023702543556e-05,
+      "loss": 0.796,
+      "step": 10788
+    },
+    {
+      "epoch": 0.1774828570491158,
+      "grad_norm": 0.7011746168136597,
+      "learning_rate": 3.8359326504270984e-05,
+      "loss": 0.8003,
+      "step": 10819
+    },
+    {
+      "epoch": 0.17799140391745136,
+      "grad_norm": 0.8060140013694763,
+      "learning_rate": 3.828826522492255e-05,
+      "loss": 0.8061,
+      "step": 10850
+    },
+    {
+      "epoch": 0.17849995078578693,
+      "grad_norm": 0.7718575596809387,
+      "learning_rate": 3.821705398930713e-05,
+      "loss": 0.8036,
+      "step": 10881
+    },
+    {
+      "epoch": 0.17900849765412252,
+      "grad_norm": 0.7823585271835327,
+      "learning_rate": 3.814569360103385e-05,
+      "loss": 0.8106,
+      "step": 10912
+    },
+    {
+      "epoch": 0.1795170445224581,
+      "grad_norm": 0.7017777562141418,
+      "learning_rate": 3.807418486539499e-05,
+      "loss": 0.8003,
+      "step": 10943
+    },
+    {
+      "epoch": 0.18002559139079366,
+      "grad_norm": 0.7048197984695435,
+      "learning_rate": 3.80025285893569e-05,
+      "loss": 0.8029,
+      "step": 10974
+    },
+    {
+      "epoch": 0.18053413825912923,
+      "grad_norm": 0.8400728702545166,
+      "learning_rate": 3.793072558155093e-05,
+      "loss": 0.805,
+      "step": 11005
+    },
+    {
+      "epoch": 0.18104268512746483,
+      "grad_norm": 0.7969333529472351,
+      "learning_rate": 3.785877665226426e-05,
+      "loss": 0.8056,
+      "step": 11036
+    },
+    {
+      "epoch": 0.1815512319958004,
+      "grad_norm": 0.6908184289932251,
+      "learning_rate": 3.778668261343079e-05,
+      "loss": 0.8039,
+      "step": 11067
+    },
+    {
+      "epoch": 0.18205977886413596,
+      "grad_norm": 0.6934013366699219,
+      "learning_rate": 3.771444427862192e-05,
+      "loss": 0.8008,
+      "step": 11098
+    },
+    {
+      "epoch": 0.18256832573247153,
+      "grad_norm": 0.7259966135025024,
+      "learning_rate": 3.7642062463037465e-05,
+      "loss": 0.7957,
+      "step": 11129
+    },
+    {
+      "epoch": 0.1830768726008071,
+      "grad_norm": 0.7105860114097595,
+      "learning_rate": 3.7569537983496373e-05,
+      "loss": 0.7957,
+      "step": 11160
+    },
+    {
+      "epoch": 0.1835854194691427,
+      "grad_norm": 0.6623875498771667,
+      "learning_rate": 3.749687165842753e-05,
+      "loss": 0.8021,
+      "step": 11191
+    },
+    {
+      "epoch": 0.18409396633747827,
+      "grad_norm": 0.808876097202301,
+      "learning_rate": 3.7424064307860536e-05,
+      "loss": 0.8129,
+      "step": 11222
+    },
+    {
+      "epoch": 0.18460251320581383,
+      "grad_norm": 0.755043089389801,
+      "learning_rate": 3.735111675341645e-05,
+      "loss": 0.8068,
+      "step": 11253
+    },
+    {
+      "epoch": 0.1851110600741494,
+      "grad_norm": 0.7748656272888184,
+      "learning_rate": 3.7278029818298524e-05,
+      "loss": 0.8131,
+      "step": 11284
+    },
+    {
+      "epoch": 0.185619606942485,
+      "grad_norm": 0.6566419005393982,
+      "learning_rate": 3.720480432728287e-05,
+      "loss": 0.8037,
+      "step": 11315
+    },
+    {
+      "epoch": 0.18612815381082057,
+      "grad_norm": 0.6518455147743225,
+      "learning_rate": 3.71314411067092e-05,
+      "loss": 0.8069,
+      "step": 11346
+    },
+    {
+      "epoch": 0.18663670067915614,
+      "grad_norm": 0.7275470495223999,
+      "learning_rate": 3.70579409844715e-05,
+      "loss": 0.802,
+      "step": 11377
+    },
+    {
+      "epoch": 0.1871452475474917,
+      "grad_norm": 0.7270357012748718,
+      "learning_rate": 3.698430479000865e-05,
+      "loss": 0.8014,
+      "step": 11408
+    },
+    {
+      "epoch": 0.1876537944158273,
+      "grad_norm": 0.6800552606582642,
+      "learning_rate": 3.691053335429509e-05,
+      "loss": 0.7999,
+      "step": 11439
+    },
+    {
+      "epoch": 0.18816234128416287,
+      "grad_norm": 0.7155413031578064,
+      "learning_rate": 3.683662750983147e-05,
+      "loss": 0.8042,
+      "step": 11470
+    },
+    {
+      "epoch": 0.18867088815249844,
+      "grad_norm": 0.7547886371612549,
+      "learning_rate": 3.676258809063518e-05,
+      "loss": 0.8036,
+      "step": 11501
+    },
+    {
+      "epoch": 0.189179435020834,
+      "grad_norm": 0.7590459585189819,
+      "learning_rate": 3.6688415932231004e-05,
+      "loss": 0.8002,
+      "step": 11532
+    },
+    {
+      "epoch": 0.1896879818891696,
+      "grad_norm": 0.7052833437919617,
+      "learning_rate": 3.661411187164166e-05,
+      "loss": 0.8012,
+      "step": 11563
+    },
+    {
+      "epoch": 0.19019652875750517,
+      "grad_norm": 0.8168420791625977,
+      "learning_rate": 3.65396767473784e-05,
+      "loss": 0.8153,
+      "step": 11594
+    },
+    {
+      "epoch": 0.19070507562584074,
+      "grad_norm": 0.6579718589782715,
+      "learning_rate": 3.6465111399431465e-05,
+      "loss": 0.8044,
+      "step": 11625
+    },
+    {
+      "epoch": 0.1912136224941763,
+      "grad_norm": 0.7866760492324829,
+      "learning_rate": 3.6390416669260674e-05,
+      "loss": 0.8033,
+      "step": 11656
+    },
+    {
+      "epoch": 0.1917221693625119,
+      "grad_norm": 0.6293924450874329,
+      "learning_rate": 3.63155933997859e-05,
+      "loss": 0.7945,
+      "step": 11687
+    },
+    {
+      "epoch": 0.19223071623084748,
+      "grad_norm": 0.8991072773933411,
+      "learning_rate": 3.624064243537758e-05,
+      "loss": 0.7971,
+      "step": 11718
+    },
+    {
+      "epoch": 0.19273926309918304,
+      "grad_norm": 0.7595002055168152,
+      "learning_rate": 3.616556462184716e-05,
+      "loss": 0.8051,
+      "step": 11749
+    },
+    {
+      "epoch": 0.1932478099675186,
+      "grad_norm": 0.6522486209869385,
+      "learning_rate": 3.609036080643755e-05,
+      "loss": 0.8089,
+      "step": 11780
+    },
+    {
+      "epoch": 0.19375635683585418,
+      "grad_norm": 0.7773357629776001,
+      "learning_rate": 3.60150318378136e-05,
+      "loss": 0.794,
+      "step": 11811
+    },
+    {
+      "epoch": 0.19426490370418978,
+      "grad_norm": 0.5722661018371582,
+      "learning_rate": 3.5939578566052465e-05,
+      "loss": 0.8024,
+      "step": 11842
+    },
+    {
+      "epoch": 0.19477345057252535,
+      "grad_norm": 0.6683876514434814,
+      "learning_rate": 3.586400184263408e-05,
+      "loss": 0.807,
+      "step": 11873
+    },
+    {
+      "epoch": 0.19528199744086092,
+      "grad_norm": 0.7072450518608093,
+      "learning_rate": 3.578830252043148e-05,
+      "loss": 0.8025,
+      "step": 11904
+    },
+    {
+      "epoch": 0.19579054430919648,
+      "grad_norm": 0.884564995765686,
+      "learning_rate": 3.571248145370125e-05,
+      "loss": 0.7972,
+      "step": 11935
+    },
+    {
+      "epoch": 0.19629909117753208,
+      "grad_norm": 1.0629959106445312,
+      "learning_rate": 3.5636539498073794e-05,
+      "loss": 0.8089,
+      "step": 11966
+    },
+    {
+      "epoch": 0.19680763804586765,
+      "grad_norm": 0.6376608610153198,
+      "learning_rate": 3.556047751054378e-05,
+      "loss": 0.8012,
+      "step": 11997
+    },
+    {
+      "epoch": 0.19731618491420322,
+      "grad_norm": 0.6260228157043457,
+      "learning_rate": 3.548429634946039e-05,
+      "loss": 0.7978,
+      "step": 12028
+    },
+    {
+      "epoch": 0.1978247317825388,
+      "grad_norm": 0.8349230289459229,
+      "learning_rate": 3.540799687451768e-05,
+      "loss": 0.8027,
+      "step": 12059
+    },
+    {
+      "epoch": 0.19833327865087438,
+      "grad_norm": 0.5764767527580261,
+      "learning_rate": 3.533157994674485e-05,
+      "loss": 0.7966,
+      "step": 12090
+    },
+    {
+      "epoch": 0.19884182551920995,
+      "grad_norm": 0.7782206535339355,
+      "learning_rate": 3.5255046428496546e-05,
+      "loss": 0.8068,
+      "step": 12121
+    },
+    {
+      "epoch": 0.19935037238754552,
+      "grad_norm": 0.6693763136863708,
+      "learning_rate": 3.517839718344311e-05,
+      "loss": 0.801,
+      "step": 12152
+    },
+    {
+      "epoch": 0.1998589192558811,
+      "grad_norm": 0.7616300582885742,
+      "learning_rate": 3.510163307656086e-05,
+      "loss": 0.8016,
+      "step": 12183
+    },
+    {
+      "epoch": 0.2003674661242167,
+      "grad_norm": 0.713007390499115,
+      "learning_rate": 3.5024754974122324e-05,
+      "loss": 0.8069,
+      "step": 12214
+    },
+    {
+      "epoch": 0.20087601299255226,
+      "grad_norm": 0.7186658382415771,
+      "learning_rate": 3.494776374368643e-05,
+      "loss": 0.8004,
+      "step": 12245
+    },
+    {
+      "epoch": 0.20138455986088782,
+      "grad_norm": 0.8170626163482666,
+      "learning_rate": 3.4870660254088724e-05,
+      "loss": 0.7951,
+      "step": 12276
+    },
+    {
+      "epoch": 0.2018931067292234,
+      "grad_norm": 0.7292721271514893,
+      "learning_rate": 3.479344537543164e-05,
+      "loss": 0.7931,
+      "step": 12307
+    },
+    {
+      "epoch": 0.20240165359755896,
+      "grad_norm": 0.726301372051239,
+      "learning_rate": 3.4716119979074565e-05,
+      "loss": 0.7968,
+      "step": 12338
+    },
+    {
+      "epoch": 0.20291020046589456,
+      "grad_norm": 0.7788697481155396,
+      "learning_rate": 3.463868493762412e-05,
+      "loss": 0.79,
+      "step": 12369
+    },
+    {
+      "epoch": 0.20341874733423013,
+      "grad_norm": 0.8112415671348572,
+      "learning_rate": 3.456114112492418e-05,
+      "loss": 0.8025,
+      "step": 12400
+    },
+    {
+      "epoch": 0.2039272942025657,
+      "grad_norm": 0.7649407982826233,
+      "learning_rate": 3.4483489416046164e-05,
+      "loss": 0.7944,
+      "step": 12431
+    },
+    {
+      "epoch": 0.20443584107090126,
+      "grad_norm": 0.7772629261016846,
+      "learning_rate": 3.440573068727905e-05,
+      "loss": 0.8003,
+      "step": 12462
+    },
+    {
+      "epoch": 0.20494438793923686,
+      "grad_norm": 0.7811031341552734,
+      "learning_rate": 3.4327865816119495e-05,
+      "loss": 0.8002,
+      "step": 12493
+    },
+    {
+      "epoch": 0.20545293480757243,
+      "grad_norm": 0.7882823944091797,
+      "learning_rate": 3.4249895681262025e-05,
+      "loss": 0.802,
+      "step": 12524
+    },
+    {
+      "epoch": 0.205961481675908,
+      "grad_norm": 0.7304728031158447,
+      "learning_rate": 3.417182116258899e-05,
+      "loss": 0.8014,
+      "step": 12555
+    },
+    {
+      "epoch": 0.20647002854424357,
+      "grad_norm": 0.7437602877616882,
+      "learning_rate": 3.409364314116074e-05,
+      "loss": 0.8059,
+      "step": 12586
+    },
+    {
+      "epoch": 0.20697857541257916,
+      "grad_norm": 0.7569501996040344,
+      "learning_rate": 3.401536249920559e-05,
+      "loss": 0.7971,
+      "step": 12617
+    },
+    {
+      "epoch": 0.20748712228091473,
+      "grad_norm": 0.6882267594337463,
+      "learning_rate": 3.393698012010998e-05,
+      "loss": 0.7959,
+      "step": 12648
+    },
+    {
+      "epoch": 0.2079956691492503,
+      "grad_norm": 0.7867191433906555,
+      "learning_rate": 3.385849688840839e-05,
+      "loss": 0.8023,
+      "step": 12679
+    },
+    {
+      "epoch": 0.20850421601758587,
+      "grad_norm": 0.7310858368873596,
+      "learning_rate": 3.3779913689773414e-05,
+      "loss": 0.7906,
+      "step": 12710
+    },
+    {
+      "epoch": 0.20901276288592147,
+      "grad_norm": 1.3257224559783936,
+      "learning_rate": 3.370123141100578e-05,
+      "loss": 0.8019,
+      "step": 12741
+    },
+    {
+      "epoch": 0.20952130975425703,
+      "grad_norm": 0.6054298877716064,
+      "learning_rate": 3.3622450940024305e-05,
+      "loss": 0.7863,
+      "step": 12772
+    },
+    {
+      "epoch": 0.2100298566225926,
+      "grad_norm": 0.7246784567832947,
+      "learning_rate": 3.35435731658559e-05,
+      "loss": 0.8038,
+      "step": 12803
+    },
+    {
+      "epoch": 0.21053840349092817,
+      "grad_norm": 0.5992689728736877,
+      "learning_rate": 3.346459897862552e-05,
+      "loss": 0.7869,
+      "step": 12834
+    },
+    {
+      "epoch": 0.21104695035926377,
+      "grad_norm": 0.813422441482544,
+      "learning_rate": 3.338552926954613e-05,
+      "loss": 0.796,
+      "step": 12865
+    },
+    {
+      "epoch": 0.21155549722759934,
+      "grad_norm": 0.7040281891822815,
+      "learning_rate": 3.330636493090868e-05,
+      "loss": 0.8024,
+      "step": 12896
+    },
+    {
+      "epoch": 0.2120640440959349,
+      "grad_norm": 0.6306488513946533,
+      "learning_rate": 3.322710685607193e-05,
+      "loss": 0.7938,
+      "step": 12927
+    },
+    {
+      "epoch": 0.21257259096427047,
+      "grad_norm": 0.7233445048332214,
+      "learning_rate": 3.314775593945251e-05,
+      "loss": 0.8005,
+      "step": 12958
+    },
+    {
+      "epoch": 0.21308113783260604,
+      "grad_norm": 0.6896563172340393,
+      "learning_rate": 3.3068313076514714e-05,
+      "loss": 0.7953,
+      "step": 12989
+    },
+    {
+      "epoch": 0.21358968470094164,
+      "grad_norm": 0.6153748631477356,
+      "learning_rate": 3.298877916376047e-05,
+      "loss": 0.8016,
+      "step": 13020
+    },
+    {
+      "epoch": 0.2140982315692772,
+      "grad_norm": 0.6767333745956421,
+      "learning_rate": 3.290915509871915e-05,
+      "loss": 0.7962,
+      "step": 13051
+    },
+    {
+      "epoch": 0.21460677843761278,
+      "grad_norm": 0.7829506993293762,
+      "learning_rate": 3.282944177993753e-05,
+      "loss": 0.8015,
+      "step": 13082
+    },
+    {
+      "epoch": 0.21511532530594835,
+      "grad_norm": 0.7379277348518372,
+      "learning_rate": 3.274964010696957e-05,
+      "loss": 0.7917,
+      "step": 13113
+    },
+    {
+      "epoch": 0.21562387217428394,
+      "grad_norm": 0.6824074387550354,
+      "learning_rate": 3.266975098036629e-05,
+      "loss": 0.7958,
+      "step": 13144
+    },
+    {
+      "epoch": 0.2161324190426195,
+      "grad_norm": 0.6095326542854309,
+      "learning_rate": 3.258977530166562e-05,
+      "loss": 0.7936,
+      "step": 13175
+    },
+    {
+      "epoch": 0.21664096591095508,
+      "grad_norm": 0.7830429673194885,
+      "learning_rate": 3.250971397338227e-05,
+      "loss": 0.8008,
+      "step": 13206
+    },
+    {
+      "epoch": 0.21714951277929065,
+      "grad_norm": 0.7481824159622192,
+      "learning_rate": 3.2429567898997404e-05,
+      "loss": 0.794,
+      "step": 13237
+    },
+    {
+      "epoch": 0.21765805964762625,
+      "grad_norm": 0.6584956645965576,
+      "learning_rate": 3.234933798294859e-05,
+      "loss": 0.7868,
+      "step": 13268
+    },
+    {
+      "epoch": 0.21816660651596181,
+      "grad_norm": 0.8784825801849365,
+      "learning_rate": 3.2269025130619535e-05,
+      "loss": 0.7951,
+      "step": 13299
+    },
+    {
+      "epoch": 0.21867515338429738,
+      "grad_norm": 0.7389800548553467,
+      "learning_rate": 3.218863024832985e-05,
+      "loss": 0.8044,
+      "step": 13330
+    },
+    {
+      "epoch": 0.21918370025263295,
+      "grad_norm": 0.7060415744781494,
+      "learning_rate": 3.2108154243324864e-05,
+      "loss": 0.7937,
+      "step": 13361
+    },
+    {
+      "epoch": 0.21969224712096855,
+      "grad_norm": 0.7183441519737244,
+      "learning_rate": 3.2027598023765345e-05,
+      "loss": 0.7932,
+      "step": 13392
+    },
+    {
+      "epoch": 0.22020079398930412,
+      "grad_norm": 0.6611085534095764,
+      "learning_rate": 3.194696249871729e-05,
+      "loss": 0.7923,
+      "step": 13423
+    },
+    {
+      "epoch": 0.22070934085763969,
+      "grad_norm": 0.7188689708709717,
+      "learning_rate": 3.186624857814164e-05,
+      "loss": 0.7942,
+      "step": 13454
+    },
+    {
+      "epoch": 0.22121788772597525,
+      "grad_norm": 0.7488799691200256,
+      "learning_rate": 3.178545717288401e-05,
+      "loss": 0.8019,
+      "step": 13485
+    },
+    {
+      "epoch": 0.22172643459431085,
+      "grad_norm": 0.8026098012924194,
+      "learning_rate": 3.170458919466444e-05,
+      "loss": 0.7847,
+      "step": 13516
+    },
+    {
+      "epoch": 0.22223498146264642,
+      "grad_norm": 0.7297918200492859,
+      "learning_rate": 3.1623645556067063e-05,
+      "loss": 0.782,
+      "step": 13547
+    },
+    {
+      "epoch": 0.222743528330982,
+      "grad_norm": 0.6407516598701477,
+      "learning_rate": 3.154262717052985e-05,
+      "loss": 0.7909,
+      "step": 13578
+    },
+    {
+      "epoch": 0.22325207519931756,
+      "grad_norm": 0.6573085188865662,
+      "learning_rate": 3.146153495233426e-05,
+      "loss": 0.798,
+      "step": 13609
+    },
+    {
+      "epoch": 0.22376062206765313,
+      "grad_norm": 0.6651787757873535,
+      "learning_rate": 3.1380369816594944e-05,
+      "loss": 0.7991,
+      "step": 13640
+    },
+    {
+      "epoch": 0.22426916893598872,
+      "grad_norm": 0.7072126269340515,
+      "learning_rate": 3.129913267924946e-05,
+      "loss": 0.7958,
+      "step": 13671
+    },
+    {
+      "epoch": 0.2247777158043243,
+      "grad_norm": 0.6639351844787598,
+      "learning_rate": 3.121782445704782e-05,
+      "loss": 0.7959,
+      "step": 13702
+    },
+    {
+      "epoch": 0.22528626267265986,
+      "grad_norm": 0.6752071976661682,
+      "learning_rate": 3.11364460675423e-05,
+      "loss": 0.7856,
+      "step": 13733
+    },
+    {
+      "epoch": 0.22579480954099543,
+      "grad_norm": 0.7549706101417542,
+      "learning_rate": 3.1054998429076934e-05,
+      "loss": 0.7804,
+      "step": 13764
+    },
+    {
+      "epoch": 0.22630335640933102,
+      "grad_norm": 0.753256618976593,
+      "learning_rate": 3.097348246077728e-05,
+      "loss": 0.7869,
+      "step": 13795
+    },
+    {
+      "epoch": 0.2268119032776666,
+      "grad_norm": 0.7454065084457397,
+      "learning_rate": 3.0891899082539924e-05,
+      "loss": 0.7919,
+      "step": 13826
+    },
+    {
+      "epoch": 0.22732045014600216,
+      "grad_norm": 0.6620218753814697,
+      "learning_rate": 3.0810249215022233e-05,
+      "loss": 0.7904,
+      "step": 13857
+    },
+    {
+      "epoch": 0.22782899701433773,
+      "grad_norm": 0.6351463794708252,
+      "learning_rate": 3.0728533779631865e-05,
+      "loss": 0.7965,
+      "step": 13888
+    },
+    {
+      "epoch": 0.22833754388267333,
+      "grad_norm": 0.7052971124649048,
+      "learning_rate": 3.064675369851637e-05,
+      "loss": 0.7907,
+      "step": 13919
+    },
+    {
+      "epoch": 0.2288460907510089,
+      "grad_norm": 0.7578268647193909,
+      "learning_rate": 3.056490989455289e-05,
+      "loss": 0.7806,
+      "step": 13950
+    },
+    {
+      "epoch": 0.22935463761934446,
+      "grad_norm": 0.6826913356781006,
+      "learning_rate": 3.0483003291337596e-05,
+      "loss": 0.7999,
+      "step": 13981
+    },
+    {
+      "epoch": 0.22986318448768003,
+      "grad_norm": 0.7119861245155334,
+      "learning_rate": 3.040103481317539e-05,
+      "loss": 0.7826,
+      "step": 14012
+    },
+    {
+      "epoch": 0.23037173135601563,
+      "grad_norm": 0.7406447529792786,
+      "learning_rate": 3.03190053850694e-05,
+      "loss": 0.8025,
+      "step": 14043
+    },
+    {
+      "epoch": 0.2308802782243512,
+      "grad_norm": 0.770252525806427,
+      "learning_rate": 3.0236915932710573e-05,
+      "loss": 0.7773,
+      "step": 14074
+    },
+    {
+      "epoch": 0.23138882509268677,
+      "grad_norm": 0.6395714282989502,
+      "learning_rate": 3.0154767382467232e-05,
+      "loss": 0.785,
+      "step": 14105
+    },
+    {
+      "epoch": 0.23189737196102234,
+      "grad_norm": 0.6831480860710144,
+      "learning_rate": 3.0072560661374582e-05,
+      "loss": 0.7875,
+      "step": 14136
+    },
+    {
+      "epoch": 0.2324059188293579,
+      "grad_norm": 0.7098875045776367,
+      "learning_rate": 2.999029669712431e-05,
+      "loss": 0.787,
+      "step": 14167
+    },
+    {
+      "epoch": 0.2329144656976935,
+      "grad_norm": 0.6381131410598755,
+      "learning_rate": 2.990797641805408e-05,
+      "loss": 0.7833,
+      "step": 14198
+    },
+    {
+      "epoch": 0.23342301256602907,
+      "grad_norm": 0.596955418586731,
+      "learning_rate": 2.982560075313704e-05,
+      "loss": 0.7855,
+      "step": 14229
+    },
+    {
+      "epoch": 0.23393155943436464,
+      "grad_norm": 0.7009894251823425,
+      "learning_rate": 2.9743170631971368e-05,
+      "loss": 0.7878,
+      "step": 14260
+    },
+    {
+      "epoch": 0.2344401063027002,
+      "grad_norm": 0.6335345506668091,
+      "learning_rate": 2.9660686984769792e-05,
+      "loss": 0.789,
+      "step": 14291
+    },
+    {
+      "epoch": 0.2349486531710358,
+      "grad_norm": 0.7286679744720459,
+      "learning_rate": 2.9578150742349047e-05,
+      "loss": 0.7829,
+      "step": 14322
+    },
+    {
+      "epoch": 0.23545720003937137,
+      "grad_norm": 0.6701176762580872,
+      "learning_rate": 2.949556283611942e-05,
+      "loss": 0.7849,
+      "step": 14353
+    },
+    {
+      "epoch": 0.23596574690770694,
+      "grad_norm": 0.66264808177948,
+      "learning_rate": 2.9412924198074206e-05,
+      "loss": 0.7885,
+      "step": 14384
+    },
+    {
+      "epoch": 0.2364742937760425,
+      "grad_norm": 0.667914092540741,
+      "learning_rate": 2.9330235760779208e-05,
+      "loss": 0.7778,
+      "step": 14415
+    },
+    {
+      "epoch": 0.2369828406443781,
+      "grad_norm": 0.8150202631950378,
+      "learning_rate": 2.9247498457362188e-05,
+      "loss": 0.7853,
+      "step": 14446
+    },
+    {
+      "epoch": 0.23749138751271368,
+      "grad_norm": 0.5859449505805969,
+      "learning_rate": 2.9164713221502373e-05,
+      "loss": 0.7864,
+      "step": 14477
+    },
+    {
+      "epoch": 0.23799993438104924,
+      "grad_norm": 0.7201155424118042,
+      "learning_rate": 2.9081880987419912e-05,
+      "loss": 0.7916,
+      "step": 14508
+    },
+    {
+      "epoch": 0.2385084812493848,
+      "grad_norm": 0.7527777552604675,
+      "learning_rate": 2.8999002689865296e-05,
+      "loss": 0.7936,
+      "step": 14539
+    },
+    {
+      "epoch": 0.2390170281177204,
+      "grad_norm": 0.6069589257240295,
+      "learning_rate": 2.8916079264108852e-05,
+      "loss": 0.7859,
+      "step": 14570
+    },
+    {
+      "epoch": 0.23952557498605598,
+      "grad_norm": 0.7169609665870667,
+      "learning_rate": 2.883311164593017e-05,
+      "loss": 0.7924,
+      "step": 14601
+    },
+    {
+      "epoch": 0.24003412185439155,
+      "grad_norm": 0.8076769113540649,
+      "learning_rate": 2.875010077160754e-05,
+      "loss": 0.782,
+      "step": 14632
+    },
+    {
+      "epoch": 0.24054266872272712,
+      "grad_norm": 0.6854205131530762,
+      "learning_rate": 2.866704757790741e-05,
+      "loss": 0.782,
+      "step": 14663
+    },
+    {
+      "epoch": 0.2410512155910627,
+      "grad_norm": 0.7560341954231262,
+      "learning_rate": 2.858395300207376e-05,
+      "loss": 0.7817,
+      "step": 14694
+    },
+    {
+      "epoch": 0.24155976245939828,
+      "grad_norm": 0.7103577852249146,
+      "learning_rate": 2.8500817981817607e-05,
+      "loss": 0.7768,
+      "step": 14725
+    },
+    {
+      "epoch": 0.24206830932773385,
+      "grad_norm": 0.817848801612854,
+      "learning_rate": 2.8417643455306336e-05,
+      "loss": 0.792,
+      "step": 14756
+    },
+    {
+      "epoch": 0.24257685619606942,
+      "grad_norm": 0.6548598408699036,
+      "learning_rate": 2.8334430361153185e-05,
+      "loss": 0.782,
+      "step": 14787
+    },
+    {
+      "epoch": 0.243085403064405,
+      "grad_norm": 0.6912696361541748,
+      "learning_rate": 2.8251179638406612e-05,
+      "loss": 0.7861,
+      "step": 14818
+    },
+    {
+      "epoch": 0.24359394993274058,
+      "grad_norm": 0.6805686354637146,
+      "learning_rate": 2.8167892226539704e-05,
+      "loss": 0.7894,
+      "step": 14849
+    },
+    {
+      "epoch": 0.24410249680107615,
+      "grad_norm": 0.5849490761756897,
+      "learning_rate": 2.8084569065439588e-05,
+      "loss": 0.7907,
+      "step": 14880
+    },
+    {
+      "epoch": 0.24461104366941172,
+      "grad_norm": 0.7393238544464111,
+      "learning_rate": 2.8001211095396807e-05,
+      "loss": 0.7883,
+      "step": 14911
+    },
+    {
+      "epoch": 0.2451195905377473,
+      "grad_norm": 0.690922737121582,
+      "learning_rate": 2.791781925709473e-05,
+      "loss": 0.7941,
+      "step": 14942
+    },
+    {
+      "epoch": 0.24562813740608289,
+      "grad_norm": 0.656465470790863,
+      "learning_rate": 2.7834394491598908e-05,
+      "loss": 0.7848,
+      "step": 14973
+    },
+    {
+      "epoch": 0.24613668427441845,
+      "grad_norm": 0.792340874671936,
+      "learning_rate": 2.7750937740346485e-05,
+      "loss": 0.7797,
+      "step": 15004
+    },
+    {
+      "epoch": 0.24664523114275402,
+      "grad_norm": 0.8570295572280884,
+      "learning_rate": 2.7667449945135564e-05,
+      "loss": 0.7858,
+      "step": 15035
+    },
+    {
+      "epoch": 0.2471537780110896,
+      "grad_norm": 0.732994019985199,
+      "learning_rate": 2.7583932048114557e-05,
+      "loss": 0.7948,
+      "step": 15066
+    },
+    {
+      "epoch": 0.2476623248794252,
+      "grad_norm": 0.6685490012168884,
+      "learning_rate": 2.7500384991771587e-05,
+      "loss": 0.785,
+      "step": 15097
+    },
+    {
+      "epoch": 0.24817087174776076,
+      "grad_norm": 0.663078784942627,
+      "learning_rate": 2.7416809718923825e-05,
+      "loss": 0.7907,
+      "step": 15128
+    },
+    {
+      "epoch": 0.24867941861609633,
+      "grad_norm": 0.6979624629020691,
+      "learning_rate": 2.7333207172706864e-05,
+      "loss": 0.7828,
+      "step": 15159
+    },
+    {
+      "epoch": 0.2491879654844319,
+      "grad_norm": 0.7100898623466492,
+      "learning_rate": 2.7249578296564088e-05,
+      "loss": 0.7806,
+      "step": 15190
+    },
+    {
+      "epoch": 0.2496965123527675,
+      "grad_norm": 0.6383324861526489,
+      "learning_rate": 2.7165924034235973e-05,
+      "loss": 0.788,
+      "step": 15221
+    },
+    {
+      "epoch": 0.25020505922110303,
+      "grad_norm": 0.7812869548797607,
+      "learning_rate": 2.708224532974953e-05,
+      "loss": 0.7828,
+      "step": 15252
+    }
+  ],
+  "logging_steps": 31,
+  "max_steps": 30517,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 7630,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.060677039050195e+19,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-15260/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ccd50401a2375309265c34189611a5ae09454df790077a35872105ac2d8457b
+size 5240

checkpoint-22890/config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "Qwen/Qwen2.5-7B",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 131072,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.0.dev0",
+  "use_cache": true,
+  "use_mrope": false,
+  "use_sliding_window": false,
+  "vocab_size": 152064
+}

checkpoint-22890/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "max_new_tokens": 2048,
+  "transformers_version": "4.44.0.dev0"
+}

checkpoint-22890/model-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f9daa802216433a0cd9df715af8c9749ce287c5912a234a617ed177e22af200
+size 4976687216

checkpoint-22890/model-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:faab92c079dee70573da2aa38bf83c1a2a2858fbee7104c30f77e30eb62ed67f
+size 4778622352

checkpoint-22890/model-00003-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d579560f2723d4a62b22ef6ec0d935dfbc2ddf0f14fc9efeac83bf5b586bf9d6
+size 4932743960

checkpoint-22890/model-00004-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:089cf99962fd351ceb590220ea351f1d18937b35376077982b276416f768317a
+size 4932743992

checkpoint-22890/model-00005-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b61bee4a8b5e41f744ec0820def58c39c0c333e6787b7e24146468348f082a0
+size 4998852296

checkpoint-22890/model-00006-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b69c0fc2847b3d693109731b3ab54a157c9139cb45579f5a6f0d266d9b4b4c1
+size 3662865184

checkpoint-22890/model-00007-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa0a1feb11bddb9c87fbb06fb2d259ea801cfb88805895e083c4ee8638696535
+size 2179989632

checkpoint-22890/model.safetensors.index.json

@@ -0,0 +1,346 @@
+{
+  "metadata": {
+    "total_size": 30462466048
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00007-of-00007.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.norm.weight": "model-00006-of-00007.safetensors"
+  }
+}

checkpoint-22890/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86c80b5735d618d171bbee209bb871d329fa1b8e2b75a19a63d0dbd888539f10
+size 16177880918

checkpoint-22890/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
+size 14244

checkpoint-22890/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9cceae98d85860224f83926a9c2d0797a9f4411abc8a933e66f23226e9ba794a
+size 1064

checkpoint-22890/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ccd50401a2375309265c34189611a5ae09454df790077a35872105ac2d8457b
+size 5240

checkpoint-30517/config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "Qwen/Qwen2.5-7B",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 131072,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.0.dev0",
+  "use_cache": true,
+  "use_mrope": false,
+  "use_sliding_window": false,
+  "vocab_size": 152064
+}

checkpoint-30517/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "max_new_tokens": 2048,
+  "transformers_version": "4.44.0.dev0"
+}

checkpoint-30517/model-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa5cfcc647c3b93c15bd0ae0029576ede7169dea9ff453985cdd0010627103ff
+size 4976687216

checkpoint-30517/model-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:faab92c079dee70573da2aa38bf83c1a2a2858fbee7104c30f77e30eb62ed67f
+size 4778622352

checkpoint-30517/model-00003-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d579560f2723d4a62b22ef6ec0d935dfbc2ddf0f14fc9efeac83bf5b586bf9d6
+size 4932743960

checkpoint-30517/model-00004-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:089cf99962fd351ceb590220ea351f1d18937b35376077982b276416f768317a
+size 4932743992

checkpoint-30517/model-00005-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b61bee4a8b5e41f744ec0820def58c39c0c333e6787b7e24146468348f082a0
+size 4998852296

checkpoint-30517/model-00006-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a979425a7192f9c299643a648e234b2210bc4800ad7fbe7f4b57ede48f917363
+size 3662865184

checkpoint-30517/model-00007-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a05164917a7374efba6ff4edc0c7004729c36574d7bb73cf45a33c858ddec2f5
+size 2179989632

checkpoint-30517/model.safetensors.index.json

@@ -0,0 +1,346 @@
+{
+  "metadata": {
+    "total_size": 30462466048
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00007-of-00007.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00004-of-00007.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00005-of-00007.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "model.norm.weight": "model-00006-of-00007.safetensors"
+  }
+}

checkpoint-30517/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f98f00bd41b6586966ca8d87365fa3ec4f04f43e67cb8b2f0f4c4c619de2ca3
+size 16177880918

checkpoint-30517/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d9cd6a0487226e5bd30d1846894c82af483733ab4381b75bae9c0745e05d405
+size 14244

checkpoint-30517/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d91c597306f26392e8da0d33a09fdcca77cfdeb5ad42248045521772fa3d64e
+size 1064

checkpoint-30517/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ccd50401a2375309265c34189611a5ae09454df790077a35872105ac2d8457b
+size 5240

config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "Qwen/Qwen2.5-7B",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 131072,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.44.0.dev0",
+  "use_cache": true,
+  "use_mrope": false,
+  "use_sliding_window": false,
+  "vocab_size": 152064
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "max_new_tokens": 2048,
+  "transformers_version": "4.44.0.dev0"
+}

model-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa5cfcc647c3b93c15bd0ae0029576ede7169dea9ff453985cdd0010627103ff
+size 4976687216